Add Peer Manager

The Peer Manager structure is responsible for maintaining a live
directory of connected CBF/Non-CBF peers. It can send and receive
messages from one or multiple peers.

It manages an underlying address manager to fetch network addresses,
and it maintains a ban score for each peer. A threshold banscore (100)
is used to signal banning of a peer. Once a peer is banned it will not
be connected to in subsequent runs.

Peer manager will be used to handle parallel fetching of filter data
from multiple peers.

.github/workflows/cont_integration.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         rust:
-          - 1.51.0 # STABLE
+          - 1.53.0 # STABLE
           - 1.46.0 # MSRV
         features:
           - default
@@ -22,6 +22,8 @@ jobs:
           - compact_filters
           - esplora,key-value-db,electrum
           - compiler
+          - rpc
+          - verify
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -75,25 +77,14 @@ jobs:
 
   test-blockchains:
     name: Test ${{ matrix.blockchain.name }}
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
         blockchain:
           - name: electrum
-            container: bitcoindevkit/electrs
-            start: /root/electrs --network regtest --jsonrpc-import
+          - name: rpc
           - name: esplora
-            container: bitcoindevkit/esplora
-            start: /root/electrs --network regtest -vvv --cookie admin:passw --jsonrpc-import --electrum-rpc-addr=0.0.0.0:60401 --http-addr 0.0.0.0:3002
-    container: ${{ matrix.blockchain.container }}
-    env:
-      BDK_RPC_AUTH: USER_PASS
-      BDK_RPC_USER: admin
-      BDK_RPC_PASS: passw
-      BDK_RPC_URL: 127.0.0.1:18443
-      BDK_RPC_WALLET: bdk-test
-      BDK_ELECTRUM_URL: tcp://127.0.0.1:60401
-      BDK_ESPLORA_URL: http://127.0.0.1:3002
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -102,25 +93,18 @@ jobs:
         with:
           path: |
             ~/.cargo/registry
+            ~/.cargo/bitcoin
+            ~/.cargo/electrs
             ~/.cargo/git
             target
           key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
-      - name: get pkg-config # running eslpora tests seems to need this
-        run: apt update && apt install -y --fix-missing pkg-config libssl-dev
-      - name: Install rustup
-        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - name: Set default toolchain
-        run: $HOME/.cargo/bin/rustup default 1.51.0 # STABLE
-      - name: Set profile
-        run: $HOME/.cargo/bin/rustup set profile minimal
-      - name: Update toolchain
-        run: $HOME/.cargo/bin/rustup update
-      - name: Start core
-        run: ./ci/start-core.sh
-      - name: start ${{ matrix.blockchain.name }}
-        run: nohup ${{ matrix.blockchain.start }} & sleep 5
+      - name: Setup rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
       - name: Test
-        run: $HOME/.cargo/bin/cargo test --features ${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests
+        run: cargo test --features test-${{ matrix.blockchain.name }} ${{ matrix.blockchain.name }}::bdk_blockchain_tests
 
   check-wasm:
     name: Check WASM
@@ -145,7 +129,7 @@ jobs:
       - run: sudo apt-get update || exit 1
       - run: sudo apt-get install -y libclang-common-10-dev clang-10 libc6-dev-i386 || exit 1
       - name: Set default toolchain
-        run: rustup default 1.51.0 # STABLE
+        run: rustup default 1.53.0 # STABLE
       - name: Set profile
         run: rustup set profile minimal
       - name: Add target wasm32

CHANGELOG.md

@@ -6,6 +6,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Wallet
+
+- Removed and replaced `set_single_recipient` with more general `drain_to` and replaced `maintain_single_recipient` with `allow_shrinking`.
+
+### Blockchain
+
+- Removed `stop_gap` from `Blockchain` trait and added it to only `ElectrumBlockchain` and `EsploraBlockchain` structs  
+
+## [v0.9.0] - [v0.8.0]
+
+### Wallet
+
+- Added Bitcoin core RPC added as blockchain backend
+- Added a `verify` feature that can be enable to verify the unconfirmed txs we download against the consensus rules
+
 ## [v0.8.0] - [v0.7.0]
 
 ### Wallet
@@ -339,7 +354,7 @@ final transaction is created by calling `finish` on the builder.
 - Use `MemoryDatabase` in the compiler example
 - Make the REPL return JSON
 
-[unreleased]: https://github.com/bitcoindevkit/bdk/compare/v0.4.0...HEAD
+[unreleased]: https://github.com/bitcoindevkit/bdk/compare/v0.9.0...HEAD
 [0.1.0-beta.1]: https://github.com/bitcoindevkit/bdk/compare/96c87ea5...0.1.0-beta.1
 [v0.2.0]: https://github.com/bitcoindevkit/bdk/compare/0.1.0-beta.1...v0.2.0
 [v0.3.0]: https://github.com/bitcoindevkit/bdk/compare/v0.2.0...v0.3.0
@@ -348,3 +363,5 @@ final transaction is created by calling `finish` on the builder.
 [v0.5.1]: https://github.com/bitcoindevkit/bdk/compare/v0.5.0...v0.5.1
 [v0.6.0]: https://github.com/bitcoindevkit/bdk/compare/v0.5.1...v0.6.0
 [v0.7.0]: https://github.com/bitcoindevkit/bdk/compare/v0.6.0...v0.7.0
+[v0.8.0]: https://github.com/bitcoindevkit/bdk/compare/v0.7.0...v0.8.0
+[v0.9.0]: https://github.com/bitcoindevkit/bdk/compare/v0.8.0...v0.9.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk"
-version = "0.8.0"
+version = "0.9.1-dev"
 edition = "2018"
 authors = ["Alekos Filini <alekos.filini@gmail.com>", "Riccardo Casatta <riccardo@casatta.it>"]
 homepage = "https://bitcoindevkit.org"
@@ -15,7 +15,7 @@ license = "MIT OR Apache-2.0"
 bdk-macros = "^0.4"
 log = "^0.4"
 miniscript = "5.1"
-bitcoin = { version = "^0.26", features = ["use-serde"] }
+bitcoin = { version = "~0.26.2", features = ["use-serde", "base64"] }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
 rand = "^0.7"
@@ -31,10 +31,11 @@ cc = { version = ">=1.0.64", optional = true }
 socks = { version = "0.3", optional = true }
 lazy_static = { version = "1.4", optional = true }
 tiny-bip39 = { version = "^0.8", optional = true }
+zeroize = { version = "<1.4.0", optional = true }
+bitcoinconsensus = { version = "0.19.0-3", optional = true }
 
 # Needed by bdk_blockchain_tests macro
-bitcoincore-rpc = {  version = "0.13", optional = true }
-serial_test = { version = "0.4", optional = true }
+bitcoincore-rpc = { version = "0.13", optional = true }
 
 # Platform-specific dependencies
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -48,6 +49,7 @@ rand = { version = "^0.7", features = ["wasm-bindgen"] }
 [features]
 minimal = []
 compiler = ["miniscript/compiler"]
+verify = ["bitcoinconsensus"]
 default = ["key-value-db", "electrum"]
 electrum = ["electrum-client"]
 esplora = ["reqwest", "futures"]
@@ -55,18 +57,22 @@ compact_filters = ["rocksdb", "socks", "lazy_static", "cc"]
 key-value-db = ["sled"]
 async-interface = ["async-trait"]
 all-keys = ["keys-bip39"]
-keys-bip39 = ["tiny-bip39"]
+keys-bip39 = ["tiny-bip39", "zeroize"]
+rpc = ["bitcoincore-rpc"]
+
 
 # Debug/Test features
 test-blockchains = ["bitcoincore-rpc", "electrum-client"]
+test-electrum = ["electrum", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-rpc = ["rpc", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-esplora = ["esplora", "electrsd/legacy", "electrsd/esplora_a33e97e1", "test-blockchains"]
 test-md-docs = ["electrum"]
 
 [dev-dependencies]
 lazy_static = "1.4"
 env_logger = "0.7"
-base64 = "^0.11"
 clap = "2.33"
-serial_test = "0.4"
+electrsd = { version= "0.6", features = ["trigger", "bitcoind_0_21_1"] }
 
 [[example]]
 name = "address_validator"
@@ -82,6 +88,6 @@ required-features = ["compiler"]
 [workspace]
 members = ["macros"]
 [package.metadata.docs.rs]
-features = ["compiler", "electrum", "esplora", "compact_filters", "key-value-db", "all-keys"]
+features = ["compiler", "electrum", "esplora", "compact_filters", "rpc", "key-value-db", "all-keys", "verify"]
 # defines the configuration attribute `docsrs`
 rustdoc-args = ["--cfg", "docsrs"]

README.md

@@ -151,6 +151,25 @@ fn main() -> Result<(), bdk::Error> {
 }
 ```
 
+## Testing
+
+### Unit testing
+
+```
+cargo test
+```
+
+### Integration testing
+
+Integration testing require testing features, for example:
+
+```
+cargo test --features test-electrum
+```
+
+The other options are `test-esplora` or `test-rpc`.
+Note that `electrs` and `bitcoind` binaries are automatically downloaded (on mac and linux), to specify you already have installed binaries you must use `--no-default-features` and provide `BITCOIND_EXE` and `ELECTRS_EXE` as environment variables.
+
 ## License
 
 Licensed under either of

ci/start-core.sh

@@ -1,13 +1,14 @@
 #!/usr/bin/env sh
 
 echo "Starting bitcoin node."
-/root/bitcoind -regtest -server -daemon -fallbackfee=0.0002 -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -blockfilterindex=1 -peerblockfilters=1
+mkdir $GITHUB_WORKSPACE/.bitcoin
+/root/bitcoind -regtest -server -daemon -datadir=$GITHUB_WORKSPACE/.bitcoin -fallbackfee=0.0002 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -blockfilterindex=1 -peerblockfilters=1
 
 echo "Waiting for bitcoin node."
-until /root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS getblockchaininfo; do
+until /root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin getblockchaininfo; do
     sleep 1
 done
-/root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS createwallet $BDK_RPC_WALLET
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin createwallet $BDK_RPC_WALLET
 echo "Generating 150 bitcoin blocks."
-ADDR=$(/root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS -rpcwallet=$BDK_RPC_WALLET getnewaddress)
-/root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS generatetoaddress 150 $ADDR
+ADDR=$(/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin -rpcwallet=$BDK_RPC_WALLET getnewaddress)
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin generatetoaddress 150 $ADDR

run_blockchain_tests.sh

@@ -1,64 +0,0 @@
-#!/bin/sh
-
-usage() {
-    cat <<'EOF'
-Script for running the bdk blockchain tests for a specific blockchain by starting up the backend in docker.
-
-Usage: ./run_blockchain_tests.sh [esplora|electrum] [test name].
-
-EOF
-}
-
-eprintln(){
-    echo "$@" >&2
-}
-
-cleanup() {
-    if test "$id"; then
-        eprintln "cleaning up $blockchain docker container $id";
-        docker rm -fv "$id" > /dev/null;
-    fi
-    trap - EXIT INT
-}
-
-# Makes sure we clean up the container at the end or if ^C
-trap 'rc=$?; cleanup; exit $rc' EXIT INT
-
-blockchain="$1"
-test_name="$2"
-
-case "$blockchain" in
-    electrum)
-        eprintln "starting electrs docker container"
-        id="$(docker run -d -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp bitcoindevkit/electrs)"
-        ;;
-    esplora)
-        eprintln "starting esplora docker container"
-        id="$(docker run -d -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp -p 127.0.0.1:3002:3002/tcp bitcoindevkit/esplora)"
-        export BDK_ESPLORA_URL=http://127.0.0.1:3002
-        ;;
-    *)
-        usage;
-        exit 1;
-        ;;
-    esac
-
-# taken from https://github.com/bitcoindevkit/bitcoin-regtest-box
-export BDK_RPC_AUTH=USER_PASS
-export BDK_RPC_USER=admin
-export BDK_RPC_PASS=passw
-export BDK_RPC_URL=127.0.0.1:18443
-export BDK_RPC_WALLET=bdk-test
-export BDK_ELECTRUM_URL=tcp://127.0.0.1:60401
-
-cli(){
-    docker exec -it "$id" /root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw $@
-}
-
-eprintln "running getwalletinfo until bitcoind seems to be alive"
-while ! cli getwalletinfo >/dev/null; do sleep 1; done
-
-# sleep again for good measure!
-sleep 1;
-
-cargo test --features "test-blockchains,$blockchain" --no-default-features "$blockchain::bdk_blockchain_tests::$test_name"

src/blockchain/any.rs

@@ -39,7 +39,7 @@
 //!
 //! # #[cfg(feature = "esplora")]
 //! # {
-//! let esplora_blockchain = EsploraBlockchain::new("...", None);
+//! let esplora_blockchain = EsploraBlockchain::new("...", None, 20);
 //! let wallet_esplora: Wallet<AnyBlockchain, _> = Wallet::new(
 //!     "...",
 //!     None,
@@ -126,31 +126,17 @@ impl Blockchain for AnyBlockchain {
 
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(impl_inner_method!(
-            self,
-            setup,
-            stop_gap,
-            database,
-            progress_update
-        ))
+        maybe_await!(impl_inner_method!(self, setup, database, progress_update))
     }
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(impl_inner_method!(
-            self,
-            sync,
-            stop_gap,
-            database,
-            progress_update
-        ))
+        maybe_await!(impl_inner_method!(self, sync, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
@@ -188,7 +174,8 @@ impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilt
 ///     r#"{
 ///    "type" : "electrum",
 ///    "url" : "ssl://electrum.blockstream.info:50002",
-///    "retry": 2
+///    "retry": 2,
+///    "stop_gap": 20
 /// }"#,
 /// )
 /// .unwrap();
@@ -198,7 +185,8 @@ impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilt
 ///         url: "ssl://electrum.blockstream.info:50002".into(),
 ///         retry: 2,
 ///         socks5: None,
-///         timeout: None
+///         timeout: None,
+///         stop_gap: 20,
 ///     })
 /// );
 /// # }

src/blockchain/compact_filters/address_manager.rs

@@ -0,0 +1,764 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Rajarshi Maitra <rajarshi149@gmail.com>
+//                    John Cantrell <johncantrell97@protonmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+use std::fs::File;
+use std::io::prelude::*;
+
+use std::collections::HashSet;
+use std::collections::VecDeque;
+use std::sync::{
+    mpsc::{channel, Receiver, SendError, Sender},
+    Arc, RwLock,
+};
+use std::thread::{self, JoinHandle};
+use std::time::Duration;
+
+use std::net::{SocketAddr, ToSocketAddrs};
+
+use std::sync::PoisonError;
+use std::sync::{MutexGuard, RwLockReadGuard, RwLockWriteGuard, WaitTimeoutResult};
+
+use serde::{Deserialize, Serialize};
+use serde_json::Error as SerdeError;
+
+use super::{Mempool, Peer, PeerError};
+
+use bitcoin::network::{
+    constants::{Network, ServiceFlags},
+    message::NetworkMessage,
+    Address,
+};
+
+/// Default address pool minimums
+const MIN_CBF_BUFFER: usize = 5;
+const MIN_NONCBF_BUFFER: usize = 5;
+
+/// A Discovery structure used by workers
+///
+/// Discovery can be initiated via a cache,
+/// Or it will start with default hardcoded seeds
+pub struct AddressDiscovery {
+    pending: VecDeque<SocketAddr>,
+    visited: HashSet<SocketAddr>,
+}
+
+impl AddressDiscovery {
+    fn new(network: Network, seeds: VecDeque<SocketAddr>) -> AddressDiscovery {
+        let mut network_seeds = AddressDiscovery::seeds(network);
+        let mut total_seeds = seeds;
+        total_seeds.append(&mut network_seeds);
+        AddressDiscovery {
+            pending: total_seeds,
+            visited: HashSet::new(),
+        }
+    }
+
+    fn add_pendings(&mut self, addresses: Vec<SocketAddr>) {
+        for addr in addresses {
+            if !self.pending.contains(&addr) && !self.visited.contains(&addr) {
+                self.pending.push_back(addr);
+            }
+        }
+    }
+
+    fn get_next(&mut self) -> Option<SocketAddr> {
+        match self.pending.pop_front() {
+            None => None,
+            Some(next) => {
+                self.visited.insert(next);
+                Some(next)
+            }
+        }
+    }
+
+    fn seeds(network: Network) -> VecDeque<SocketAddr> {
+        let mut seeds = VecDeque::new();
+
+        let port: u16 = match network {
+            Network::Bitcoin => 8333,
+            Network::Testnet => 18333,
+            Network::Regtest => 18444,
+            Network::Signet => 38333,
+        };
+
+        let seedhosts: &[&str] = match network {
+            Network::Bitcoin => &[
+                "seed.bitcoin.sipa.be",
+                "dnsseed.bluematt.me",
+                "dnsseed.bitcoin.dashjr.org",
+                "seed.bitcoinstats.com",
+                "seed.bitcoin.jonasschnelli.ch",
+                "seed.btc.petertodd.org",
+                "seed.bitcoin.sprovoost.nl",
+                "dnsseed.emzy.de",
+                "seed.bitcoin.wiz.biz",
+            ],
+            Network::Testnet => &[
+                "testnet-seed.bitcoin.jonasschnelli.ch",
+                "seed.tbtc.petertodd.org",
+                "seed.testnet.bitcoin.sprovoost.nl",
+                "testnet-seed.bluematt.me",
+            ],
+            Network::Regtest => &[],
+            Network::Signet => &[],
+        };
+
+        for seedhost in seedhosts.iter() {
+            if let Ok(lookup) = (*seedhost, port).to_socket_addrs() {
+                for host in lookup {
+                    if host.is_ipv4() {
+                        seeds.push_back(host);
+                    }
+                }
+            }
+        }
+        seeds
+    }
+}
+
+/// Crawler structure that will interface with Discovery and public bitcoin network
+///
+/// Address manager will spawn multiple crawlers in separate threads to discover new addresses.
+struct AddressWorker {
+    discovery: Arc<RwLock<AddressDiscovery>>,
+    sender: Sender<(SocketAddr, ServiceFlags)>,
+    network: Network,
+}
+
+impl AddressWorker {
+    fn new(
+        discovery: Arc<RwLock<AddressDiscovery>>,
+        sender: Sender<(SocketAddr, ServiceFlags)>,
+        network: Network,
+    ) -> AddressWorker {
+        AddressWorker {
+            discovery,
+            sender,
+            network,
+        }
+    }
+
+    fn try_receive_addr(&mut self, peer: &Peer) -> Result<(), AddressManagerError> {
+        if let Some(NetworkMessage::Addr(new_addresses)) =
+            peer.recv("addr", Some(Duration::from_secs(1)))?
+        {
+            self.consume_addr(new_addresses)?;
+        }
+
+        Ok(())
+    }
+
+    fn consume_addr(&mut self, addrs: Vec<(u32, Address)>) -> Result<(), AddressManagerError> {
+        let mut discovery_lock = self.discovery.write().map_err(PeerError::from)?;
+        let mut addresses = Vec::new();
+        for network_addrs in addrs {
+            if let Ok(socket_addrs) = network_addrs.1.socket_addr() {
+                addresses.push(socket_addrs);
+            }
+        }
+        discovery_lock.add_pendings(addresses);
+
+        Ok(())
+    }
+
+    fn work(&mut self) -> Result<(), AddressManagerError> {
+        loop {
+            let next_address = {
+                let mut address_discovery = self.discovery.write()?;
+                address_discovery.get_next()
+            };
+
+            match next_address {
+                Some(address) => {
+                    let potential_peer = Peer::connect_with_timeout(
+                        address,
+                        Duration::from_secs(1),
+                        Arc::new(Mempool::default()),
+                        self.network,
+                    );
+
+                    if let Ok(peer) = potential_peer {
+                        peer.send(NetworkMessage::GetAddr)?;
+                        self.try_receive_addr(&peer)?;
+                        self.try_receive_addr(&peer)?;
+                        self.sender.send((address, peer.get_version().services))?;
+                        // TODO: Investigate why close is being called on non existent connections
+                        // currently the errors are ignored
+                        peer.close().unwrap_or(());
+                    }
+                }
+                None => continue,
+            }
+        }
+    }
+}
+
+/// A dedicated cache structure, with cbf/non_cbf separation
+///
+/// [AddressCache] will interface with file i/o
+/// And can te turned into seeds. Generation of seed will put previously cached
+/// cbf addresses at front of the vec, to boost up cbf node findings
+#[derive(Serialize, Deserialize)]
+struct AddressCache {
+    banned_peers: HashSet<SocketAddr>,
+    cbf: HashSet<SocketAddr>,
+    non_cbf: HashSet<SocketAddr>,
+}
+
+impl AddressCache {
+    fn empty() -> Self {
+        Self {
+            banned_peers: HashSet::new(),
+            cbf: HashSet::new(),
+            non_cbf: HashSet::new(),
+        }
+    }
+
+    fn from_file(path: &str) -> Result<Option<Self>, AddressManagerError> {
+        let serialized: Result<String, _> = std::fs::read_to_string(path);
+        let serialized = match serialized {
+            Ok(contents) => contents,
+            Err(_) => return Ok(None),
+        };
+
+        let address_cache = serde_json::from_str(&serialized)?;
+        Ok(Some(address_cache))
+    }
+
+    fn write_to_file(&self, path: &str) -> Result<(), AddressManagerError> {
+        let serialized = serde_json::to_string_pretty(&self)?;
+
+        let mut cache_file = File::create(path)?;
+
+        cache_file.write_all(serialized.as_bytes())?;
+
+        Ok(())
+    }
+
+    fn make_seeds(&self) -> VecDeque<SocketAddr> {
+        self.cbf
+            .iter()
+            .chain(self.non_cbf.iter())
+            .copied()
+            .collect()
+    }
+
+    fn remove_address(&mut self, addrs: &SocketAddr, cbf: bool) -> bool {
+        if cbf {
+            self.cbf.remove(addrs)
+        } else {
+            self.non_cbf.remove(addrs)
+        }
+    }
+
+    fn add_address(&mut self, addrs: SocketAddr, cbf: bool) -> bool {
+        if cbf {
+            self.cbf.insert(addrs)
+        } else {
+            self.non_cbf.insert(addrs)
+        }
+    }
+
+    fn add_to_banlist(&mut self, addrs: SocketAddr, cbf: bool) {
+        if self.banned_peers.insert(addrs) {
+            self.remove_address(&addrs, cbf);
+        }
+    }
+}
+
+/// A Live directory maintained by [AddressManager] of freshly found cbf and non_cbf nodes by workers
+///
+/// Each instance of new [AddressManager] with have fresh [AddressDirectory]
+/// This is independent from the cache and will be an in-memory database to
+/// fetch addresses to the user.  
+struct AddressDirectory {
+    cbf_nodes: HashSet<SocketAddr>,
+    non_cbf_nodes: HashSet<SocketAddr>,
+
+    // List of addresses it has previously provided to the caller (PeerManager)
+    previously_sent: HashSet<SocketAddr>,
+}
+
+impl AddressDirectory {
+    fn new() -> AddressDirectory {
+        AddressDirectory {
+            cbf_nodes: HashSet::new(),
+            non_cbf_nodes: HashSet::new(),
+            previously_sent: HashSet::new(),
+        }
+    }
+
+    fn add_address(&mut self, addr: SocketAddr, cbf: bool) {
+        if cbf {
+            self.cbf_nodes.insert(addr);
+        } else {
+            self.non_cbf_nodes.insert(addr);
+        }
+    }
+
+    fn get_new_address(&mut self, cbf: bool) -> Option<SocketAddr> {
+        if cbf {
+            if let Some(new_addresses) = self
+                .cbf_nodes
+                .iter()
+                .filter(|item| !self.previously_sent.contains(item))
+                .collect::<Vec<&SocketAddr>>()
+                .pop()
+            {
+                self.previously_sent.insert(*new_addresses);
+                Some(*new_addresses)
+            } else {
+                None
+            }
+        } else if let Some(new_addresses) = self
+            .non_cbf_nodes
+            .iter()
+            .filter(|item| !self.previously_sent.contains(item))
+            .collect::<Vec<&SocketAddr>>()
+            .pop()
+        {
+            self.previously_sent.insert(*new_addresses);
+            Some(*new_addresses)
+        } else {
+            None
+        }
+    }
+
+    fn get_cbf_address_count(&self) -> usize {
+        self.cbf_nodes.len()
+    }
+
+    fn get_non_cbf_address_count(&self) -> usize {
+        self.non_cbf_nodes.len()
+    }
+
+    fn remove_address(&mut self, addrs: &SocketAddr, cbf: bool) {
+        if cbf {
+            self.cbf_nodes.remove(addrs);
+        } else {
+            self.non_cbf_nodes.remove(addrs);
+        }
+    }
+
+    fn get_cbf_buffer(&self) -> usize {
+        self.cbf_nodes
+            .iter()
+            .filter(|item| !self.previously_sent.contains(item))
+            .count()
+    }
+
+    fn get_non_cbf_buffer(&self) -> usize {
+        self.non_cbf_nodes
+            .iter()
+            .filter(|item| !self.previously_sent.contains(item))
+            .count()
+    }
+}
+
+/// Discovery statistics, useful for logging
+#[derive(Clone, Copy)]
+pub struct DiscoveryData {
+    queued: usize,
+    visited: usize,
+    non_cbf_count: usize,
+    cbf_count: usize,
+}
+
+/// Progress trait for discovery statistics logging
+pub trait DiscoveryProgress {
+    /// Update progress
+    fn update(&self, data: DiscoveryData);
+}
+
+/// Used when progress updates are not desired
+#[derive(Clone)]
+pub struct NoDiscoveryProgress;
+
+impl DiscoveryProgress for NoDiscoveryProgress {
+    fn update(&self, _data: DiscoveryData) {}
+}
+
+/// Used to log progress update
+#[derive(Clone)]
+pub struct LogDiscoveryProgress;
+
+impl DiscoveryProgress for LogDiscoveryProgress {
+    fn update(&self, data: DiscoveryData) {
+        log::trace!(
+            "P2P Discovery: {} queued, {} visited, {} connected, {} cbf_enabled",
+            data.queued,
+            data.visited,
+            data.non_cbf_count,
+            data.cbf_count
+        );
+
+        #[cfg(test)]
+        println!(
+            "P2P Discovery: {} queued, {} visited, {} connected, {} cbf_enabled",
+            data.queued, data.visited, data.non_cbf_count, data.cbf_count
+        );
+    }
+}
+
+/// A manager structure managing address discovery
+///
+/// Manager will try to maintain a given address buffer in its directory
+/// buffer = len(exiting addresses) - len(previously provided addresses)
+/// Manager will crawl the network until buffer criteria is satisfied
+/// Manager will bootstrap workers from a cache, to speed up discovery progress in
+/// subsequent call after the first crawl.
+/// Manager will keep track of the cache and only update it if previously
+/// unknown addresses are found.
+pub struct AddressManager<P: DiscoveryProgress> {
+    directory: AddressDirectory,
+    cache_filename: String,
+    discovery: Arc<RwLock<AddressDiscovery>>,
+    threads: usize,
+    receiver: Receiver<(SocketAddr, ServiceFlags)>,
+    sender: Sender<(SocketAddr, ServiceFlags)>,
+    network: Network,
+    cbf_buffer: usize,
+    non_cbf_buffer: usize,
+    progress: P,
+}
+
+impl<P: DiscoveryProgress> AddressManager<P> {
+    /// Create a new manager. Initiate Discovery seeds from the cache
+    /// if it exists, else start with hardcoded seeds
+    pub fn new(
+        network: Network,
+        cache_filename: String,
+        threads: usize,
+        cbf_buffer: Option<usize>,
+        non_cbf_buffer: Option<usize>,
+        progress: P,
+    ) -> Result<AddressManager<P>, AddressManagerError> {
+        let (sender, receiver) = channel();
+
+        let seeds = match AddressCache::from_file(&cache_filename)? {
+            Some(cache) => cache.make_seeds(),
+            None => VecDeque::new(),
+        };
+
+        let min_cbf = cbf_buffer.unwrap_or(MIN_CBF_BUFFER);
+
+        let min_non_cbf = non_cbf_buffer.unwrap_or(MIN_NONCBF_BUFFER);
+
+        let discovery = AddressDiscovery::new(network, seeds);
+
+        Ok(AddressManager {
+            cache_filename,
+            directory: AddressDirectory::new(),
+            discovery: Arc::new(RwLock::new(discovery)),
+            sender,
+            receiver,
+            network,
+            threads,
+            cbf_buffer: min_cbf,
+            non_cbf_buffer: min_non_cbf,
+            progress,
+        })
+    }
+
+    /// Get running address discovery progress
+    fn get_progress(&self) -> Result<DiscoveryData, AddressManagerError> {
+        let (queued_count, visited_count) = {
+            let address_discovery = self.discovery.read()?;
+            (
+                address_discovery.pending.len(),
+                address_discovery.visited.len(),
+            )
+        };
+
+        let cbf_node_count = self.directory.get_cbf_address_count();
+        let other_node_count = self.directory.get_non_cbf_address_count();
+
+        Ok(DiscoveryData {
+            queued: queued_count,
+            visited: visited_count,
+            non_cbf_count: cbf_node_count + other_node_count,
+            cbf_count: cbf_node_count,
+        })
+    }
+
+    /// Spawn [self.thread] no. of worker threads
+    fn spawn_workers(&mut self) -> Vec<JoinHandle<()>> {
+        let mut worker_handles: Vec<JoinHandle<()>> = vec![];
+        for _ in 0..self.threads {
+            let sender = self.sender.clone();
+            let discovery = self.discovery.clone();
+            let network = self.network;
+            let worker_handle = thread::spawn(move || {
+                let mut worker = AddressWorker::new(discovery, sender, network);
+                worker.work().unwrap();
+            });
+            worker_handles.push(worker_handle);
+        }
+        worker_handles
+    }
+
+    /// Crawl the Bitcoin network until required number of cbf/non_cbf nodes are found
+    ///
+    /// - This will start a bunch of crawlers.
+    /// - load up the existing cache.
+    /// - Update the cache with new found peers.
+    /// - check if address is in banlist
+    /// - run crawlers until buffer requirement is matched
+    /// - flush the current cache into disk
+    pub fn fetch(&mut self) -> Result<(), AddressManagerError> {
+        self.spawn_workers();
+
+        // Get already existing cache
+        let mut cache = match AddressCache::from_file(&self.cache_filename)? {
+            Some(cache) => cache,
+            None => AddressCache::empty(),
+        };
+
+        while self.directory.get_cbf_buffer() < self.cbf_buffer
+            || self.directory.get_non_cbf_buffer() < self.non_cbf_buffer
+        {
+            if let Ok(message) = self.receiver.recv() {
+                let (addr, flag) = message;
+                if !cache.banned_peers.contains(&addr) {
+                    let cbf = flag.has(ServiceFlags::COMPACT_FILTERS);
+                    self.directory.add_address(addr, cbf);
+                    cache.add_address(addr, cbf);
+                }
+            }
+        }
+
+        self.progress.update(self.get_progress()?);
+
+        // When completed, flush the cache
+        cache.write_to_file(&self.cache_filename)?;
+
+        Ok(())
+    }
+
+    /// Get a new addresses not previously provided
+    pub fn get_new_cbf_address(&mut self) -> Option<SocketAddr> {
+        self.directory.get_new_address(true)
+    }
+
+    /// Get a new non_cbf address
+    pub fn get_new_non_cbf_address(&mut self) -> Option<SocketAddr> {
+        self.directory.get_new_address(false)
+    }
+
+    /// Ban an address
+    pub fn ban_peer(&mut self, addrs: &SocketAddr, cbf: bool) -> Result<(), AddressManagerError> {
+        let mut cache = AddressCache::from_file(&self.cache_filename)?.ok_or_else(|| {
+            AddressManagerError::Generic("Address Cache file not found".to_string())
+        })?;
+
+        cache.add_to_banlist(*addrs, cbf);
+
+        // When completed, flush the cache
+        cache.write_to_file(&self.cache_filename).unwrap();
+
+        self.directory.remove_address(addrs, cbf);
+
+        Ok(())
+    }
+
+    /// Get all the known CBF addresses
+    pub fn get_known_cbfs(&self) -> Option<Vec<SocketAddr>> {
+        let addresses = self
+            .directory
+            .cbf_nodes
+            .iter()
+            .copied()
+            .collect::<Vec<SocketAddr>>();
+
+        match addresses.len() {
+            0 => None,
+            _ => Some(addresses),
+        }
+    }
+
+    /// Get all the known regular addresses
+    pub fn get_known_non_cbfs(&self) -> Option<Vec<SocketAddr>> {
+        let addresses = self
+            .directory
+            .non_cbf_nodes
+            .iter()
+            .copied()
+            .collect::<Vec<SocketAddr>>();
+
+        match addresses.len() {
+            0 => None,
+            _ => Some(addresses),
+        }
+    }
+
+    /// Get previously tried addresses
+    pub fn get_previously_tried(&self) -> Option<Vec<SocketAddr>> {
+        let addresses = self
+            .directory
+            .previously_sent
+            .iter()
+            .copied()
+            .collect::<Vec<SocketAddr>>();
+
+        match addresses.len() {
+            0 => None,
+            _ => Some(addresses),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum AddressManagerError {
+    /// Std I/O Error
+    Io(std::io::Error),
+
+    /// Internal Peer error
+    Peer(PeerError),
+
+    /// Internal Mutex poisoning error
+    MutexPoisoned,
+
+    /// Internal Mutex wait timed out
+    MutexTimedOut,
+
+    /// Internal RW read lock poisoned
+    RwReadLockPoisined,
+
+    /// Internal RW write lock poisoned
+    RwWriteLockPoisoned,
+
+    /// Internal MPSC sending error
+    MpscSendError,
+
+    /// Serde Json Error
+    SerdeJson(SerdeError),
+
+    /// Generic Errors
+    Generic(String),
+}
+
+impl_error!(PeerError, Peer, AddressManagerError);
+impl_error!(std::io::Error, Io, AddressManagerError);
+impl_error!(SerdeError, SerdeJson, AddressManagerError);
+
+impl<T> From<PoisonError<MutexGuard<'_, T>>> for AddressManagerError {
+    fn from(_: PoisonError<MutexGuard<'_, T>>) -> Self {
+        AddressManagerError::MutexPoisoned
+    }
+}
+
+impl<T> From<PoisonError<RwLockWriteGuard<'_, T>>> for AddressManagerError {
+    fn from(_: PoisonError<RwLockWriteGuard<'_, T>>) -> Self {
+        AddressManagerError::RwWriteLockPoisoned
+    }
+}
+
+impl<T> From<PoisonError<RwLockReadGuard<'_, T>>> for AddressManagerError {
+    fn from(_: PoisonError<RwLockReadGuard<'_, T>>) -> Self {
+        AddressManagerError::RwReadLockPoisined
+    }
+}
+
+impl<T> From<PoisonError<(MutexGuard<'_, T>, WaitTimeoutResult)>> for AddressManagerError {
+    fn from(err: PoisonError<(MutexGuard<'_, T>, WaitTimeoutResult)>) -> Self {
+        let (_, wait_result) = err.into_inner();
+        if wait_result.timed_out() {
+            AddressManagerError::MutexTimedOut
+        } else {
+            AddressManagerError::MutexPoisoned
+        }
+    }
+}
+
+impl<T> From<SendError<T>> for AddressManagerError {
+    fn from(_: SendError<T>) -> Self {
+        AddressManagerError::MpscSendError
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    #[ignore]
+    fn test_address_manager() {
+        // Initiate a manager with an non existent cache file name.
+        // It will create a new cache file
+        let mut manager = AddressManager::new(
+            Network::Bitcoin,
+            "addr_cache".to_string(),
+            20,
+            None,
+            None,
+            LogDiscoveryProgress,
+        )
+        .unwrap();
+
+        // start the crawlers and time them
+        println!("Starting manager and initial fetch");
+        let start = std::time::Instant::now();
+        manager.fetch().unwrap();
+        let duration1 = start.elapsed();
+        println!("Completed Initial fetch");
+
+        // Create a new manager from existing cache and fetch again
+        let mut manager = AddressManager::new(
+            Network::Bitcoin,
+            "addr_cache".to_string(),
+            20,
+            None,
+            None,
+            LogDiscoveryProgress,
+        )
+        .unwrap();
+
+        // start the crawlers and time them
+        println!("Starting new fetch with previous cache");
+        let start = std::time::Instant::now();
+        manager.fetch().unwrap();
+        let duration2 = start.elapsed();
+        println!("Completed new fetch()");
+
+        println!("Time taken for initial crawl: {:#?}", duration1);
+        println!("Time taken for next crawl {:#?}", duration2);
+
+        // Check Buffer Management
+
+        println!("Checking buffer management");
+        // Fetch few new address and ensure buffer goes to zero
+        let mut addrs_list = Vec::new();
+        for _ in 0..5 {
+            let addr_cbf = manager.get_new_cbf_address().unwrap();
+            let addrs_non_cbf = manager.get_new_non_cbf_address().unwrap();
+
+            addrs_list.push(addr_cbf);
+
+            addrs_list.push(addrs_non_cbf);
+        }
+
+        assert_eq!(addrs_list.len(), 10);
+
+        // This should exhaust the cbf buffer
+        assert_eq!(manager.directory.get_cbf_buffer(), 0);
+
+        // Calling fetch again should start crawlers until buffer
+        // requirements are matched.
+        println!("Address buffer exhausted, starting new fetch");
+        manager.fetch().unwrap();
+        println!("Fetch Complete");
+        // It should again have a cbf buffer of 5
+        assert_eq!(manager.directory.get_cbf_buffer(), 5);
+
+        println!("Buffer management passed");
+    }
+}

src/blockchain/compact_filters/mod.rs

@@ -63,7 +63,9 @@ use bitcoin::{Network, OutPoint, Transaction, Txid};
 
 use rocksdb::{Options, SliceTransform, DB};
 
+mod address_manager;
 mod peer;
+mod peermngr;
 mod store;
 mod sync;
 
@@ -71,14 +73,17 @@ use super::{Blockchain, Capability, ConfigurableBlockchain, Progress};
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
 use crate::types::{KeychainKind, LocalUtxo, TransactionDetails};
-use crate::FeeRate;
+use crate::{ConfirmationTime, FeeRate};
 
 use peer::*;
 use store::*;
 use sync::*;
 
+// Only added to avoid unused warnings in addrsmngr module
+pub use address_manager::{
+    AddressManager, DiscoveryProgress, LogDiscoveryProgress, NoDiscoveryProgress,
+};
 pub use peer::{Mempool, Peer};
-
 const SYNC_HEADERS_COST: f32 = 1.0;
 const SYNC_FILTERS_COST: f32 = 11.6 * 1_000.0;
 const PROCESS_BLOCKS_COST: f32 = 20_000.0;
@@ -146,7 +151,7 @@ impl CompactFiltersBlockchain {
         database: &mut D,
         tx: &Transaction,
         height: Option<u32>,
-        timestamp: u64,
+        timestamp: Option<u64>,
         internal_max_deriv: &mut Option<u32>,
         external_max_deriv: &mut Option<u32>,
     ) -> Result<(), Error> {
@@ -206,9 +211,9 @@ impl CompactFiltersBlockchain {
                 transaction: Some(tx.clone()),
                 received: incoming,
                 sent: outgoing,
-                height,
-                timestamp,
-                fees: inputs_sum.saturating_sub(outputs_sum),
+                confirmation_time: ConfirmationTime::new(height, timestamp),
+                verified: height.is_some(),
+                fee: Some(inputs_sum.saturating_sub(outputs_sum)),
             };
 
             info!("Saving tx {}", tx.txid);
@@ -229,7 +234,6 @@ impl Blockchain for CompactFiltersBlockchain {
     #[allow(clippy::mutex_atomic)] // Mutex is easier to understand than a CAS loop.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        _stop_gap: Option<usize>, // TODO: move to electrum and esplora only
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
@@ -364,18 +368,18 @@ impl Blockchain for CompactFiltersBlockchain {
         );
         let mut updates = database.begin_batch();
         for details in database.iter_txs(false)? {
-            match details.height {
-                Some(height) if (height as usize) < last_synced_block => continue,
+            match details.confirmation_time {
+                Some(c) if (c.height as usize) < last_synced_block => continue,
                 _ => updates.del_tx(&details.txid, false)?,
             };
         }
         database.commit_batch(updates)?;
 
         match first_peer.ask_for_mempool() {
-            Err(CompactFiltersError::PeerBloomDisabled) => {
+            Err(PeerError::PeerBloomDisabled(_)) => {
                 log::warn!("Peer has BLOOM disabled, we can't ask for the mempool")
             }
-            e => e?,
+            e => e.map_err(CompactFiltersError::from)?,
         };
 
         let mut internal_max_deriv = None;
@@ -387,18 +391,23 @@ impl Blockchain for CompactFiltersBlockchain {
                     database,
                     tx,
                     Some(height as u32),
-                    0,
+                    None,
                     &mut internal_max_deriv,
                     &mut external_max_deriv,
                 )?;
             }
         }
-        for tx in first_peer.get_mempool().iter_txs().iter() {
+        for tx in first_peer
+            .get_mempool()
+            .iter_txs()
+            .map_err(CompactFiltersError::from)?
+            .iter()
+        {
             self.process_tx(
                 database,
                 tx,
                 None,
-                0,
+                None,
                 &mut internal_max_deriv,
                 &mut external_max_deriv,
             )?;
@@ -436,11 +445,14 @@ impl Blockchain for CompactFiltersBlockchain {
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
         Ok(self.peers[0]
             .get_mempool()
-            .get_tx(&Inventory::Transaction(*txid)))
+            .get_tx(&Inventory::Transaction(*txid))
+            .map_err(CompactFiltersError::from)?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        self.peers[0].broadcast_tx(tx.clone())?;
+        self.peers[0]
+            .broadcast_tx(tx.clone())
+            .map_err(CompactFiltersError::from)?;
 
         Ok(())
     }
@@ -488,7 +500,8 @@ impl ConfigurableBlockchain for CompactFiltersBlockchain {
             .peers
             .iter()
             .map(|peer_conf| match &peer_conf.socks5 {
-                None => Peer::connect(&peer_conf.address, Arc::clone(&mempool), config.network),
+                None => Peer::connect(&peer_conf.address, Arc::clone(&mempool), config.network)
+                    .map_err(CompactFiltersError::from),
                 Some(proxy) => Peer::connect_proxy(
                     peer_conf.address.as_str(),
                     proxy,
@@ -498,7 +511,8 @@ impl ConfigurableBlockchain for CompactFiltersBlockchain {
                         .map(|(a, b)| (a.as_str(), b.as_str())),
                     Arc::clone(&mempool),
                     config.network,
-                ),
+                )
+                .map_err(CompactFiltersError::from),
             })
             .collect::<Result<_, _>>()?;
 
@@ -547,6 +561,9 @@ pub enum CompactFiltersError {
 
     /// Wrapper for [`crate::error::Error`]
     Global(Box<crate::error::Error>),
+
+    /// Internal Peer Error
+    Peer(PeerError),
 }
 
 impl fmt::Display for CompactFiltersError {
@@ -561,6 +578,7 @@ impl_error!(rocksdb::Error, Db, CompactFiltersError);
 impl_error!(std::io::Error, Io, CompactFiltersError);
 impl_error!(bitcoin::util::bip158::Error, Bip158, CompactFiltersError);
 impl_error!(std::time::SystemTimeError, Time, CompactFiltersError);
+impl_error!(PeerError, Peer, CompactFiltersError);
 
 impl From<crate::error::Error> for CompactFiltersError {
     fn from(err: crate::error::Error) -> Self {

src/blockchain/compact_filters/peer.rs

@@ -10,11 +10,15 @@
 // licenses.
 
 use std::collections::HashMap;
-use std::net::{TcpStream, ToSocketAddrs};
+use std::fmt;
+use std::net::{SocketAddr, TcpStream, ToSocketAddrs};
 use std::sync::{Arc, Condvar, Mutex, RwLock};
 use std::thread;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
+use std::sync::PoisonError;
+use std::sync::{MutexGuard, RwLockReadGuard, RwLockWriteGuard, WaitTimeoutResult};
+
 use socks::{Socks5Stream, ToTargetAddr};
 
 use rand::{thread_rng, Rng};
@@ -30,8 +34,6 @@ use bitcoin::network::stream_reader::StreamReader;
 use bitcoin::network::Address;
 use bitcoin::{Block, Network, Transaction, Txid, Wtxid};
 
-use super::CompactFiltersError;
-
 type ResponsesMap = HashMap<&'static str, Arc<(Mutex<Vec<NetworkMessage>>, Condvar)>>;
 
 pub(crate) const TIMEOUT_SECS: u64 = 30;
@@ -65,17 +67,18 @@ impl Mempool {
     ///
     /// Note that this doesn't propagate the transaction to other
     /// peers. To do that, [`broadcast`](crate::blockchain::Blockchain::broadcast) should be used.
-    pub fn add_tx(&self, tx: Transaction) {
-        let mut guard = self.0.write().unwrap();
+    pub fn add_tx(&self, tx: Transaction) -> Result<(), PeerError> {
+        let mut guard = self.0.write()?;
 
         guard.wtxids.insert(tx.wtxid(), tx.txid());
         guard.txs.insert(tx.txid(), tx);
+        Ok(())
     }
 
     /// Look-up a transaction in the mempool given an [`Inventory`] request
-    pub fn get_tx(&self, inventory: &Inventory) -> Option<Transaction> {
+    pub fn get_tx(&self, inventory: &Inventory) -> Result<Option<Transaction>, PeerError> {
         let identifer = match inventory {
-            Inventory::Error | Inventory::Block(_) | Inventory::WitnessBlock(_) => return None,
+            Inventory::Error | Inventory::Block(_) | Inventory::WitnessBlock(_) => return Ok(None),
             Inventory::Transaction(txid) => TxIdentifier::Txid(*txid),
             Inventory::WitnessTransaction(txid) => TxIdentifier::Txid(*txid),
             Inventory::WTx(wtxid) => TxIdentifier::Wtxid(*wtxid),
@@ -85,27 +88,34 @@ impl Mempool {
                     inv_type,
                     hash
                 );
-                return None;
+                return Ok(None);
             }
         };
 
         let txid = match identifer {
             TxIdentifier::Txid(txid) => Some(txid),
-            TxIdentifier::Wtxid(wtxid) => self.0.read().unwrap().wtxids.get(&wtxid).cloned(),
+            TxIdentifier::Wtxid(wtxid) => self.0.read()?.wtxids.get(&wtxid).cloned(),
         };
 
-        txid.map(|txid| self.0.read().unwrap().txs.get(&txid).cloned())
-            .flatten()
+        let result = match txid {
+            Some(txid) => {
+                let read_lock = self.0.read()?;
+                read_lock.txs.get(&txid).cloned()
+            }
+            None => None,
+        };
+
+        Ok(result)
     }
 
     /// Return whether or not the mempool contains a transaction with a given txid
-    pub fn has_tx(&self, txid: &Txid) -> bool {
-        self.0.read().unwrap().txs.contains_key(txid)
+    pub fn has_tx(&self, txid: &Txid) -> Result<bool, PeerError> {
+        Ok(self.0.read()?.txs.contains_key(txid))
     }
 
     /// Return the list of transactions contained in the mempool
-    pub fn iter_txs(&self) -> Vec<Transaction> {
-        self.0.read().unwrap().txs.values().cloned().collect()
+    pub fn iter_txs(&self) -> Result<Vec<Transaction>, PeerError> {
+        Ok(self.0.read()?.txs.values().cloned().collect())
     }
 }
 
@@ -133,12 +143,31 @@ impl Peer {
         address: A,
         mempool: Arc<Mempool>,
         network: Network,
-    ) -> Result<Self, CompactFiltersError> {
+    ) -> Result<Self, PeerError> {
         let stream = TcpStream::connect(address)?;
 
         Peer::from_stream(stream, mempool, network)
     }
 
+    /// Connect to a peer over a plaintext TCP connection with a timeout
+    ///
+    /// This function behaves exactly the same as `connect` except for two differences
+    /// 1) It assumes your ToSocketAddrs will resolve to a single address
+    /// 2) It lets you specify a connection timeout
+    pub fn connect_with_timeout<A: ToSocketAddrs>(
+        address: A,
+        timeout: Duration,
+        mempool: Arc<Mempool>,
+        network: Network,
+    ) -> Result<Self, PeerError> {
+        let socket_addr = address
+            .to_socket_addrs()?
+            .next()
+            .ok_or(PeerError::AddresseResolution)?;
+        let stream = TcpStream::connect_timeout(&socket_addr, timeout)?;
+        Peer::from_stream(stream, mempool, network)
+    }
+
     /// Connect to a peer through a SOCKS5 proxy, optionally by using some credentials, specified
     /// as a tuple of `(username, password)`
     ///
@@ -150,7 +179,7 @@ impl Peer {
         credentials: Option<(&str, &str)>,
         mempool: Arc<Mempool>,
         network: Network,
-    ) -> Result<Self, CompactFiltersError> {
+    ) -> Result<Self, PeerError> {
         let socks_stream = if let Some((username, password)) = credentials {
             Socks5Stream::connect_with_password(proxy, target, username, password)?
         } else {
@@ -165,12 +194,12 @@ impl Peer {
         stream: TcpStream,
         mempool: Arc<Mempool>,
         network: Network,
-    ) -> Result<Self, CompactFiltersError> {
+    ) -> Result<Self, PeerError> {
         let writer = Arc::new(Mutex::new(stream.try_clone()?));
         let responses: Arc<RwLock<ResponsesMap>> = Arc::new(RwLock::new(HashMap::new()));
         let connected = Arc::new(RwLock::new(true));
 
-        let mut locked_writer = writer.lock().unwrap();
+        let mut locked_writer = writer.lock()?;
 
         let reader_thread_responses = Arc::clone(&responses);
         let reader_thread_writer = Arc::clone(&writer);
@@ -185,6 +214,7 @@ impl Peer {
                 reader_thread_mempool,
                 reader_thread_connected,
             )
+            .unwrap()
         });
 
         let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() as i64;
@@ -209,46 +239,55 @@ impl Peer {
                 0,
             )),
         )?;
-        let version = if let NetworkMessage::Version(version) =
-            Self::_recv(&responses, "version", None).unwrap()
-        {
-            version
-        } else {
-            return Err(CompactFiltersError::InvalidResponse);
+
+        let version = match Self::_recv(&responses, "version", Some(Duration::from_secs(1)))? {
+            Some(NetworkMessage::Version(version)) => version,
+            _ => {
+                return Err(PeerError::InvalidResponse(locked_writer.peer_addr()?));
+            }
         };
 
-        if let NetworkMessage::Verack = Self::_recv(&responses, "verack", None).unwrap() {
+        if let Some(NetworkMessage::Verack) =
+            Self::_recv(&responses, "verack", Some(Duration::from_secs(1)))?
+        {
             Self::_send(&mut locked_writer, network.magic(), NetworkMessage::Verack)?;
         } else {
-            return Err(CompactFiltersError::InvalidResponse);
+            return Err(PeerError::InvalidResponse(locked_writer.peer_addr()?));
         }
 
         std::mem::drop(locked_writer);
 
         Ok(Peer {
             writer,
-            reader_thread,
             responses,
+            reader_thread,
             connected,
             mempool,
-            network,
             version,
+            network,
         })
     }
 
+    /// Close the peer connection
+    // Consume Self
+    pub fn close(self) -> Result<(), PeerError> {
+        let locked_writer = self.writer.lock()?;
+        Ok((*locked_writer).shutdown(std::net::Shutdown::Both)?)
+    }
+
+    /// Get the socket address of the remote peer
+    pub fn get_address(&self) -> Result<SocketAddr, PeerError> {
+        let locked_writer = self.writer.lock()?;
+        Ok(locked_writer.peer_addr()?)
+    }
+
     /// Send a Bitcoin network message
-    fn _send(
-        writer: &mut TcpStream,
-        magic: u32,
-        payload: NetworkMessage,
-    ) -> Result<(), CompactFiltersError> {
+    fn _send(writer: &mut TcpStream, magic: u32, payload: NetworkMessage) -> Result<(), PeerError> {
         log::trace!("==> {:?}", payload);
 
         let raw_message = RawNetworkMessage { magic, payload };
 
-        raw_message
-            .consensus_encode(writer)
-            .map_err(|_| CompactFiltersError::DataCorruption)?;
+        raw_message.consensus_encode(writer)?;
 
         Ok(())
     }
@@ -258,30 +297,30 @@ impl Peer {
         responses: &Arc<RwLock<ResponsesMap>>,
         wait_for: &'static str,
         timeout: Option<Duration>,
-    ) -> Option<NetworkMessage> {
+    ) -> Result<Option<NetworkMessage>, PeerError> {
         let message_resp = {
-            let mut lock = responses.write().unwrap();
+            let mut lock = responses.write()?;
             let message_resp = lock.entry(wait_for).or_default();
             Arc::clone(&message_resp)
         };
 
         let (lock, cvar) = &*message_resp;
 
-        let mut messages = lock.lock().unwrap();
+        let mut messages = lock.lock()?;
         while messages.is_empty() {
             match timeout {
-                None => messages = cvar.wait(messages).unwrap(),
+                None => messages = cvar.wait(messages)?,
                 Some(t) => {
-                    let result = cvar.wait_timeout(messages, t).unwrap();
+                    let result = cvar.wait_timeout(messages, t)?;
                     if result.1.timed_out() {
-                        return None;
+                        return Ok(None);
                     }
                     messages = result.0;
                 }
             }
         }
 
-        messages.pop()
+        Ok(messages.pop())
     }
 
     /// Return the [`VersionMessage`] sent by the peer
@@ -300,8 +339,8 @@ impl Peer {
     }
 
     /// Return whether or not the peer is still connected
-    pub fn is_connected(&self) -> bool {
-        *self.connected.read().unwrap()
+    pub fn is_connected(&self) -> Result<bool, PeerError> {
+        Ok(*self.connected.read()?)
     }
 
     /// Internal function called once the `reader_thread` is spawned
@@ -312,14 +351,14 @@ impl Peer {
         reader_thread_writer: Arc<Mutex<TcpStream>>,
         reader_thread_mempool: Arc<Mempool>,
         reader_thread_connected: Arc<RwLock<bool>>,
-    ) {
+    ) -> Result<(), PeerError> {
         macro_rules! check_disconnect {
             ($call:expr) => {
                 match $call {
                     Ok(good) => good,
                     Err(e) => {
                         log::debug!("Error {:?}", e);
-                        *reader_thread_connected.write().unwrap() = false;
+                        *reader_thread_connected.write()? = false;
 
                         break;
                     }
@@ -328,7 +367,7 @@ impl Peer {
         }
 
         let mut reader = StreamReader::new(connection, None);
-        loop {
+        while *reader_thread_connected.read()? {
             let raw_message: RawNetworkMessage = check_disconnect!(reader.read_next());
 
             let in_message = if raw_message.magic != network.magic() {
@@ -342,7 +381,7 @@ impl Peer {
             match in_message {
                 NetworkMessage::Ping(nonce) => {
                     check_disconnect!(Self::_send(
-                        &mut reader_thread_writer.lock().unwrap(),
+                        &mut *reader_thread_writer.lock()?,
                         network.magic(),
                         NetworkMessage::Pong(nonce),
                     ));
@@ -353,19 +392,21 @@ impl Peer {
                 NetworkMessage::GetData(ref inv) => {
                     let (found, not_found): (Vec<_>, Vec<_>) = inv
                         .iter()
-                        .map(|item| (*item, reader_thread_mempool.get_tx(item)))
+                        .map(|item| (*item, reader_thread_mempool.get_tx(item).unwrap()))
                         .partition(|(_, d)| d.is_some());
                     for (_, found_tx) in found {
                         check_disconnect!(Self::_send(
-                            &mut reader_thread_writer.lock().unwrap(),
+                            &mut *reader_thread_writer.lock()?,
                             network.magic(),
-                            NetworkMessage::Tx(found_tx.unwrap()),
+                            NetworkMessage::Tx(found_tx.ok_or_else(|| PeerError::Generic(
+                                "Got None while expecting Transaction".to_string()
+                            ))?),
                         ));
                     }
 
                     if !not_found.is_empty() {
                         check_disconnect!(Self::_send(
-                            &mut reader_thread_writer.lock().unwrap(),
+                            &mut *reader_thread_writer.lock()?,
                             network.magic(),
                             NetworkMessage::NotFound(
                                 not_found.into_iter().map(|(i, _)| i).collect(),
@@ -377,21 +418,23 @@ impl Peer {
             }
 
             let message_resp = {
-                let mut lock = reader_thread_responses.write().unwrap();
+                let mut lock = reader_thread_responses.write()?;
                 let message_resp = lock.entry(in_message.cmd()).or_default();
                 Arc::clone(&message_resp)
             };
 
             let (lock, cvar) = &*message_resp;
-            let mut messages = lock.lock().unwrap();
+            let mut messages = lock.lock()?;
             messages.push(in_message);
             cvar.notify_all();
         }
+
+        Ok(())
     }
 
     /// Send a raw Bitcoin message to the peer
-    pub fn send(&self, payload: NetworkMessage) -> Result<(), CompactFiltersError> {
-        let mut writer = self.writer.lock().unwrap();
+    pub fn send(&self, payload: NetworkMessage) -> Result<(), PeerError> {
+        let mut writer = self.writer.lock()?;
         Self::_send(&mut writer, self.network.magic(), payload)
     }
 
@@ -400,30 +443,27 @@ impl Peer {
         &self,
         wait_for: &'static str,
         timeout: Option<Duration>,
-    ) -> Result<Option<NetworkMessage>, CompactFiltersError> {
-        Ok(Self::_recv(&self.responses, wait_for, timeout))
+    ) -> Result<Option<NetworkMessage>, PeerError> {
+        Self::_recv(&self.responses, wait_for, timeout)
     }
 }
 
 pub trait CompactFiltersPeer {
-    fn get_cf_checkpt(
-        &self,
-        filter_type: u8,
-        stop_hash: BlockHash,
-    ) -> Result<CFCheckpt, CompactFiltersError>;
+    fn get_cf_checkpt(&self, filter_type: u8, stop_hash: BlockHash)
+        -> Result<CFCheckpt, PeerError>;
     fn get_cf_headers(
         &self,
         filter_type: u8,
         start_height: u32,
         stop_hash: BlockHash,
-    ) -> Result<CFHeaders, CompactFiltersError>;
+    ) -> Result<CFHeaders, PeerError>;
     fn get_cf_filters(
         &self,
         filter_type: u8,
         start_height: u32,
         stop_hash: BlockHash,
-    ) -> Result<(), CompactFiltersError>;
-    fn pop_cf_filter_resp(&self) -> Result<CFilter, CompactFiltersError>;
+    ) -> Result<(), PeerError>;
+    fn pop_cf_filter_resp(&self) -> Result<CFilter, PeerError>;
 }
 
 impl CompactFiltersPeer for Peer {
@@ -431,22 +471,20 @@ impl CompactFiltersPeer for Peer {
         &self,
         filter_type: u8,
         stop_hash: BlockHash,
-    ) -> Result<CFCheckpt, CompactFiltersError> {
+    ) -> Result<CFCheckpt, PeerError> {
         self.send(NetworkMessage::GetCFCheckpt(GetCFCheckpt {
             filter_type,
             stop_hash,
         }))?;
 
-        let response = self
-            .recv("cfcheckpt", Some(Duration::from_secs(TIMEOUT_SECS)))?
-            .ok_or(CompactFiltersError::Timeout)?;
+        let response = self.recv("cfcheckpt", Some(Duration::from_secs(TIMEOUT_SECS)))?;
         let response = match response {
-            NetworkMessage::CFCheckpt(response) => response,
-            _ => return Err(CompactFiltersError::InvalidResponse),
+            Some(NetworkMessage::CFCheckpt(response)) => response,
+            _ => return Err(PeerError::InvalidResponse(self.get_address()?)),
         };
 
         if response.filter_type != filter_type {
-            return Err(CompactFiltersError::InvalidResponse);
+            return Err(PeerError::InvalidResponse(self.get_address()?));
         }
 
         Ok(response)
@@ -457,35 +495,31 @@ impl CompactFiltersPeer for Peer {
         filter_type: u8,
         start_height: u32,
         stop_hash: BlockHash,
-    ) -> Result<CFHeaders, CompactFiltersError> {
+    ) -> Result<CFHeaders, PeerError> {
         self.send(NetworkMessage::GetCFHeaders(GetCFHeaders {
             filter_type,
             start_height,
             stop_hash,
         }))?;
 
-        let response = self
-            .recv("cfheaders", Some(Duration::from_secs(TIMEOUT_SECS)))?
-            .ok_or(CompactFiltersError::Timeout)?;
+        let response = self.recv("cfheaders", Some(Duration::from_secs(TIMEOUT_SECS)))?;
         let response = match response {
-            NetworkMessage::CFHeaders(response) => response,
-            _ => return Err(CompactFiltersError::InvalidResponse),
+            Some(NetworkMessage::CFHeaders(response)) => response,
+            _ => return Err(PeerError::InvalidResponse(self.get_address()?)),
         };
 
         if response.filter_type != filter_type {
-            return Err(CompactFiltersError::InvalidResponse);
+            return Err(PeerError::InvalidResponse(self.get_address()?));
         }
 
         Ok(response)
     }
 
-    fn pop_cf_filter_resp(&self) -> Result<CFilter, CompactFiltersError> {
-        let response = self
-            .recv("cfilter", Some(Duration::from_secs(TIMEOUT_SECS)))?
-            .ok_or(CompactFiltersError::Timeout)?;
+    fn pop_cf_filter_resp(&self) -> Result<CFilter, PeerError> {
+        let response = self.recv("cfilter", Some(Duration::from_secs(TIMEOUT_SECS)))?;
         let response = match response {
-            NetworkMessage::CFilter(response) => response,
-            _ => return Err(CompactFiltersError::InvalidResponse),
+            Some(NetworkMessage::CFilter(response)) => response,
+            _ => return Err(PeerError::InvalidResponse(self.get_address()?)),
         };
 
         Ok(response)
@@ -496,7 +530,7 @@ impl CompactFiltersPeer for Peer {
         filter_type: u8,
         start_height: u32,
         stop_hash: BlockHash,
-    ) -> Result<(), CompactFiltersError> {
+    ) -> Result<(), PeerError> {
         self.send(NetworkMessage::GetCFilters(GetCFilters {
             filter_type,
             start_height,
@@ -508,13 +542,13 @@ impl CompactFiltersPeer for Peer {
 }
 
 pub trait InvPeer {
-    fn get_block(&self, block_hash: BlockHash) -> Result<Option<Block>, CompactFiltersError>;
-    fn ask_for_mempool(&self) -> Result<(), CompactFiltersError>;
-    fn broadcast_tx(&self, tx: Transaction) -> Result<(), CompactFiltersError>;
+    fn get_block(&self, block_hash: BlockHash) -> Result<Option<Block>, PeerError>;
+    fn ask_for_mempool(&self) -> Result<(), PeerError>;
+    fn broadcast_tx(&self, tx: Transaction) -> Result<(), PeerError>;
 }
 
 impl InvPeer for Peer {
-    fn get_block(&self, block_hash: BlockHash) -> Result<Option<Block>, CompactFiltersError> {
+    fn get_block(&self, block_hash: BlockHash) -> Result<Option<Block>, PeerError> {
         self.send(NetworkMessage::GetData(vec![Inventory::WitnessBlock(
             block_hash,
         )]))?;
@@ -522,51 +556,126 @@ impl InvPeer for Peer {
         match self.recv("block", Some(Duration::from_secs(TIMEOUT_SECS)))? {
             None => Ok(None),
             Some(NetworkMessage::Block(response)) => Ok(Some(response)),
-            _ => Err(CompactFiltersError::InvalidResponse),
+            _ => Err(PeerError::InvalidResponse(self.get_address()?)),
         }
     }
 
-    fn ask_for_mempool(&self) -> Result<(), CompactFiltersError> {
+    fn ask_for_mempool(&self) -> Result<(), PeerError> {
         if !self.version.services.has(ServiceFlags::BLOOM) {
-            return Err(CompactFiltersError::PeerBloomDisabled);
+            return Err(PeerError::PeerBloomDisabled(self.get_address()?));
         }
 
         self.send(NetworkMessage::MemPool)?;
         let inv = match self.recv("inv", Some(Duration::from_secs(5)))? {
             None => return Ok(()), // empty mempool
             Some(NetworkMessage::Inv(inv)) => inv,
-            _ => return Err(CompactFiltersError::InvalidResponse),
+            _ => return Err(PeerError::InvalidResponse(self.get_address()?)),
         };
 
         let getdata = inv
             .iter()
             .cloned()
             .filter(
-                |item| matches!(item, Inventory::Transaction(txid) if !self.mempool.has_tx(txid)),
+                |item| matches!(item, Inventory::Transaction(txid) if !self.mempool.has_tx(txid).unwrap()),
             )
             .collect::<Vec<_>>();
         let num_txs = getdata.len();
         self.send(NetworkMessage::GetData(getdata))?;
 
         for _ in 0..num_txs {
-            let tx = self
-                .recv("tx", Some(Duration::from_secs(TIMEOUT_SECS)))?
-                .ok_or(CompactFiltersError::Timeout)?;
+            let tx = self.recv("tx", Some(Duration::from_secs(TIMEOUT_SECS)))?;
             let tx = match tx {
-                NetworkMessage::Tx(tx) => tx,
-                _ => return Err(CompactFiltersError::InvalidResponse),
+                Some(NetworkMessage::Tx(tx)) => tx,
+                _ => return Err(PeerError::InvalidResponse(self.get_address()?)),
             };
 
-            self.mempool.add_tx(tx);
+            self.mempool.add_tx(tx)?;
         }
 
         Ok(())
     }
 
-    fn broadcast_tx(&self, tx: Transaction) -> Result<(), CompactFiltersError> {
-        self.mempool.add_tx(tx.clone());
+    fn broadcast_tx(&self, tx: Transaction) -> Result<(), PeerError> {
+        self.mempool.add_tx(tx.clone())?;
         self.send(NetworkMessage::Tx(tx))?;
 
         Ok(())
     }
 }
+
+/// Peer Errors
+#[derive(Debug)]
+pub enum PeerError {
+    /// Internal I/O error
+    Io(std::io::Error),
+
+    /// Internal system time error
+    Time(std::time::SystemTimeError),
+
+    /// A peer sent an invalid or unexpected response
+    InvalidResponse(SocketAddr),
+
+    /// Peer had bloom filter disabled
+    PeerBloomDisabled(SocketAddr),
+
+    /// Internal Mutex poisoning error
+    MutexPoisoned,
+
+    /// Internal Mutex wait timed out
+    MutexTimedout,
+
+    /// Internal RW read lock poisoned
+    RwReadLockPoisined,
+
+    /// Internal RW write lock poisoned
+    RwWriteLockPoisoned,
+
+    /// Mempool Mutex poisoned
+    MempoolPoisoned,
+
+    /// Network address resolution Error
+    AddresseResolution,
+
+    /// Generic Errors
+    Generic(String),
+}
+
+impl std::fmt::Display for PeerError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for PeerError {}
+
+impl_error!(std::io::Error, Io, PeerError);
+impl_error!(std::time::SystemTimeError, Time, PeerError);
+
+impl<T> From<PoisonError<MutexGuard<'_, T>>> for PeerError {
+    fn from(_: PoisonError<MutexGuard<'_, T>>) -> Self {
+        PeerError::MutexPoisoned
+    }
+}
+
+impl<T> From<PoisonError<RwLockWriteGuard<'_, T>>> for PeerError {
+    fn from(_: PoisonError<RwLockWriteGuard<'_, T>>) -> Self {
+        PeerError::RwWriteLockPoisoned
+    }
+}
+
+impl<T> From<PoisonError<RwLockReadGuard<'_, T>>> for PeerError {
+    fn from(_: PoisonError<RwLockReadGuard<'_, T>>) -> Self {
+        PeerError::RwReadLockPoisined
+    }
+}
+
+impl<T> From<PoisonError<(MutexGuard<'_, T>, WaitTimeoutResult)>> for PeerError {
+    fn from(err: PoisonError<(MutexGuard<'_, T>, WaitTimeoutResult)>) -> Self {
+        let (_, wait_result) = err.into_inner();
+        if wait_result.timed_out() {
+            PeerError::MutexTimedout
+        } else {
+            PeerError::MutexPoisoned
+        }
+    }
+}

src/blockchain/compact_filters/peermngr.rs

@@ -0,0 +1,578 @@
+use super::address_manager::{AddressManager, AddressManagerError, DiscoveryProgress};
+use super::peer::{Mempool, Peer, PeerError, TIMEOUT_SECS};
+
+use std::net::SocketAddr;
+use std::sync::Arc;
+use std::time;
+
+use bitcoin::network::constants::{Network, ServiceFlags};
+use bitcoin::network::message::NetworkMessage;
+
+use std::path::PathBuf;
+
+use std::collections::BTreeMap;
+
+// Peer Manager Configuration constants
+const MIN_CBF_PEERS: usize = 2;
+const MIN_TOTAL_PEERS: usize = 5;
+const MIN_CRAWLER_THREADS: usize = 20;
+const BAN_SCORE_THRESHOLD: usize = 100;
+const RECEIVE_TIMEOUT: time::Duration = time::Duration::from_secs(TIMEOUT_SECS);
+
+#[allow(dead_code)]
+/// An Error structure describing Peer Management errors
+#[derive(Debug)]
+pub enum PeerManagerError {
+    // Internal Peer Error
+    Peer(PeerError),
+
+    // Internal AddressManager Error
+    AddrsManager(AddressManagerError),
+
+    // Os String Error
+    OsString(std::ffi::OsString),
+
+    // Peer not found in directory
+    PeerNotFound,
+
+    // Generic Internal Error
+    Generic(String),
+}
+
+impl_error!(PeerError, Peer, PeerManagerError);
+impl_error!(AddressManagerError, AddrsManager, PeerManagerError);
+impl_error!(std::ffi::OsString, OsString, PeerManagerError);
+
+/// Peer Data stored in the manager's directory
+#[derive(Debug)]
+struct PeerData {
+    peer: Peer,
+    is_cbf: bool,
+    ban_score: usize,
+}
+
+#[allow(dead_code)]
+/// A Directory structure to hold live Peers
+/// All peers in the directory have live ongoing connection
+/// Banning a peer removes it from the directory
+#[derive(Default, Debug)]
+struct PeerDirectory {
+    peers: BTreeMap<SocketAddr, PeerData>,
+}
+
+#[allow(dead_code)]
+impl PeerDirectory {
+    fn new() -> Self {
+        Self::default()
+    }
+
+    fn get_cbf_peers(&self) -> Option<Vec<&PeerData>> {
+        let cbf_peers = self
+            .peers
+            .iter()
+            .filter(|(_, peer)| peer.is_cbf)
+            .map(|(_, peer)| peer)
+            .collect::<Vec<&PeerData>>();
+
+        match cbf_peers.len() {
+            0 => None,
+            _ => Some(cbf_peers),
+        }
+    }
+
+    fn get_cbf_addresses(&self) -> Option<Vec<SocketAddr>> {
+        let cbf_addrseses = self
+            .peers
+            .iter()
+            .filter_map(
+                |(addrs, peerdata)| {
+                    if peerdata.is_cbf {
+                        Some(addrs)
+                    } else {
+                        None
+                    }
+                },
+            )
+            .copied()
+            .collect::<Vec<SocketAddr>>();
+
+        match cbf_addrseses.len() {
+            0 => None,
+            _ => Some(cbf_addrseses),
+        }
+    }
+
+    fn get_non_cbf_peers(&self) -> Option<Vec<&PeerData>> {
+        let non_cbf_peers = self
+            .peers
+            .iter()
+            .filter(|(_, peerdata)| !peerdata.is_cbf)
+            .map(|(_, peerdata)| peerdata)
+            .collect::<Vec<&PeerData>>();
+
+        match non_cbf_peers.len() {
+            0 => None,
+            _ => Some(non_cbf_peers),
+        }
+    }
+
+    fn get_non_cbf_addresses(&self) -> Option<Vec<SocketAddr>> {
+        let addresses = self
+            .peers
+            .iter()
+            .filter_map(
+                |(addrs, peerdata)| {
+                    if !peerdata.is_cbf {
+                        Some(addrs)
+                    } else {
+                        None
+                    }
+                },
+            )
+            .copied()
+            .collect::<Vec<SocketAddr>>();
+
+        match addresses.len() {
+            0 => None,
+            _ => Some(addresses),
+        }
+    }
+
+    fn get_cbf_peers_mut(&mut self) -> Option<Vec<&mut PeerData>> {
+        let peers = self
+            .peers
+            .iter_mut()
+            .filter(|(_, peerdata)| peerdata.is_cbf)
+            .map(|(_, peerdata)| peerdata)
+            .collect::<Vec<&mut PeerData>>();
+
+        match peers.len() {
+            0 => None,
+            _ => Some(peers),
+        }
+    }
+
+    fn get_non_cbf_peers_mut(&mut self) -> Option<Vec<&mut PeerData>> {
+        let peers = self
+            .peers
+            .iter_mut()
+            .filter(|(_, peerdata)| !peerdata.is_cbf)
+            .map(|(_, peerdata)| peerdata)
+            .collect::<Vec<&mut PeerData>>();
+
+        match peers.len() {
+            0 => None,
+            _ => Some(peers),
+        }
+    }
+
+    fn get_cbf_count(&self) -> usize {
+        self.peers
+            .iter()
+            .filter(|(_, peerdata)| peerdata.is_cbf)
+            .count()
+    }
+
+    fn get_non_cbf_count(&self) -> usize {
+        self.peers
+            .iter()
+            .filter(|(_, peerdata)| !peerdata.is_cbf)
+            .count()
+    }
+
+    fn insert_peer(&mut self, peerdata: PeerData) -> Result<(), PeerManagerError> {
+        let addrs = peerdata.peer.get_address()?;
+        self.peers.entry(addrs).or_insert(peerdata);
+        Ok(())
+    }
+
+    fn remove_peer(&mut self, addrs: &SocketAddr) -> Option<PeerData> {
+        self.peers.remove(addrs)
+    }
+
+    fn get_peer_banscore(&self, addrs: &SocketAddr) -> Option<usize> {
+        self.peers.get(addrs).map(|peerdata| peerdata.ban_score)
+    }
+
+    fn get_peerdata_mut(&mut self, address: &SocketAddr) -> Option<&mut PeerData> {
+        self.peers.get_mut(address)
+    }
+
+    fn get_peerdata(&self, address: &SocketAddr) -> Option<&PeerData> {
+        self.peers.get(address)
+    }
+
+    fn is_cbf(&self, addrs: &SocketAddr) -> Option<bool> {
+        if let Some(peer) = self.peers.get(addrs) {
+            match peer.is_cbf {
+                true => Some(true),
+                false => Some(false),
+            }
+        } else {
+            None
+        }
+    }
+}
+
+#[allow(dead_code)]
+pub struct PeerManager<P: DiscoveryProgress> {
+    addrs_mngr: AddressManager<P>,
+    directory: PeerDirectory,
+    mempool: Arc<Mempool>,
+    min_cbf: usize,
+    min_total: usize,
+    network: Network,
+}
+
+#[allow(dead_code)]
+impl<P: DiscoveryProgress> PeerManager<P> {
+    pub fn init(
+        network: Network,
+        cache_dir: &str,
+        crawler_threads: Option<usize>,
+        progress: P,
+        cbf_peers: Option<usize>,
+        total_peers: Option<usize>,
+    ) -> Result<Self, PeerManagerError> {
+        let mut cache_filename = PathBuf::from(cache_dir);
+        cache_filename.push("addr_cache");
+
+        // Fetch minimum peer requirements, either by user input, or via default
+        let min_cbf = cbf_peers.unwrap_or(MIN_CBF_PEERS);
+
+        let min_total = total_peers.unwrap_or(MIN_TOTAL_PEERS);
+
+        let cbf_buff = min_cbf * 2;
+        let non_cbf_buff = (min_total - min_cbf) * 2;
+
+        // Create internal items
+        let addrs_mngr = AddressManager::new(
+            network,
+            cache_filename.into_os_string().into_string()?,
+            crawler_threads.unwrap_or(MIN_CRAWLER_THREADS),
+            Some(cbf_buff),
+            Some(non_cbf_buff),
+            progress,
+        )?;
+
+        let mempool = Arc::new(Mempool::new());
+
+        let peer_dir = PeerDirectory::new();
+
+        // Create self and update
+        let mut manager = Self {
+            addrs_mngr,
+            directory: peer_dir,
+            mempool,
+            min_cbf,
+            min_total,
+            network,
+        };
+
+        manager.update_directory()?;
+
+        Ok(manager)
+    }
+
+    fn update_directory(&mut self) -> Result<(), PeerManagerError> {
+        while self.directory.get_cbf_count() < self.min_cbf
+            || self.directory.get_non_cbf_count() < (self.min_total - self.min_cbf)
+        {
+            // First connect with cbf peers, then with non_cbf
+            let cbf_fetch = self.directory.get_cbf_count() < self.min_cbf;
+
+            // Try to get an address
+            // if not present start crawlers
+            let target_addrs = match cbf_fetch {
+                true => {
+                    if let Some(addrs) = self.addrs_mngr.get_new_cbf_address() {
+                        addrs
+                    } else {
+                        self.addrs_mngr.fetch()?;
+                        continue;
+                    }
+                }
+                false => {
+                    if let Some(addrs) = self.addrs_mngr.get_new_non_cbf_address() {
+                        addrs
+                    } else {
+                        self.addrs_mngr.fetch()?;
+                        continue;
+                    }
+                }
+            };
+
+            if let Ok(peer) = Peer::connect(target_addrs, Arc::clone(&self.mempool), self.network) {
+                let address = peer.get_address()?;
+
+                assert_eq!(address, target_addrs);
+
+                let is_cbf = peer
+                    .get_version()
+                    .services
+                    .has(ServiceFlags::COMPACT_FILTERS);
+
+                let peerdata = PeerData {
+                    peer,
+                    is_cbf,
+                    ban_score: 0,
+                };
+
+                self.directory.insert_peer(peerdata)?;
+            } else {
+                continue;
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn set_banscore(
+        &mut self,
+        increase_by: usize,
+        address: &SocketAddr,
+    ) -> Result<(), PeerManagerError> {
+        let mut current_score = if let Some(peer) = self.directory.get_peerdata_mut(address) {
+            peer.ban_score
+        } else {
+            return Err(PeerManagerError::PeerNotFound);
+        };
+
+        current_score += increase_by;
+
+        let mut banned = false;
+
+        if current_score >= BAN_SCORE_THRESHOLD {
+            match (
+                self.directory.is_cbf(address),
+                self.directory.remove_peer(address),
+            ) {
+                (Some(true), Some(_)) => {
+                    self.addrs_mngr.ban_peer(address, true)?;
+                    banned = true;
+                }
+                (Some(false), Some(_)) => {
+                    self.addrs_mngr.ban_peer(address, false)?;
+                    banned = true;
+                }
+                _ => {
+                    return Err(PeerManagerError::Generic(
+                        "data inconsistency in directory, should not happen".to_string(),
+                    ))
+                }
+            }
+        }
+
+        if banned {
+            self.update_directory()?;
+        }
+
+        Ok(())
+    }
+
+    pub fn send_to(
+        &self,
+        address: &SocketAddr,
+        message: NetworkMessage,
+    ) -> Result<(), PeerManagerError> {
+        if let Some(peerdata) = self.directory.get_peerdata(address) {
+            peerdata.peer.send(message)?;
+            Ok(())
+        } else {
+            Err(PeerManagerError::PeerNotFound)
+        }
+    }
+
+    pub fn receive_from(
+        &self,
+        address: &SocketAddr,
+        wait_for: &'static str,
+    ) -> Result<Option<NetworkMessage>, PeerManagerError> {
+        if let Some(peerdata) = self.directory.get_peerdata(address) {
+            if let Some(response) = peerdata.peer.recv(wait_for, Some(RECEIVE_TIMEOUT))? {
+                Ok(Some(response))
+            } else {
+                Ok(None)
+            }
+        } else {
+            Err(PeerManagerError::PeerNotFound)
+        }
+    }
+
+    pub fn connected_cbf_addresses(&self) -> Option<Vec<SocketAddr>> {
+        self.directory.get_cbf_addresses()
+    }
+
+    pub fn connected_non_cbf_addresses(&self) -> Option<Vec<SocketAddr>> {
+        self.directory.get_non_cbf_addresses()
+    }
+
+    pub fn known_cbf_addresses(&self) -> Option<Vec<SocketAddr>> {
+        self.addrs_mngr.get_known_cbfs()
+    }
+
+    pub fn known_non_cbf_addresses(&self) -> Option<Vec<SocketAddr>> {
+        self.addrs_mngr.get_known_non_cbfs()
+    }
+
+    pub fn previously_tried_addresses(&self) -> Option<Vec<SocketAddr>> {
+        self.addrs_mngr.get_previously_tried()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::LogDiscoveryProgress;
+    use super::*;
+
+    #[test]
+    #[ignore]
+    fn test_ban() {
+        let mut manager = PeerManager::init(
+            Network::Bitcoin,
+            ".",
+            None,
+            LogDiscoveryProgress,
+            None,
+            None,
+        )
+        .unwrap();
+
+        let connected_cbfs = manager.connected_cbf_addresses().unwrap();
+        let connected_non_cbfs = manager.connected_non_cbf_addresses().unwrap();
+
+        println!("Currently Connected CBFs: {:#?}", connected_cbfs);
+        assert_eq!(connected_cbfs.len(), 2);
+        assert_eq!(connected_non_cbfs.len(), 3);
+
+        let to_banned = &connected_cbfs[0];
+
+        println!("Banning address : {}", to_banned);
+
+        manager.set_banscore(100, to_banned).unwrap();
+
+        let newly_connected = manager.connected_cbf_addresses().unwrap();
+
+        println!("Newly Connected CBFs: {:#?}", newly_connected);
+
+        assert_eq!(newly_connected.len(), 2);
+
+        assert_ne!(newly_connected, connected_cbfs);
+    }
+
+    #[test]
+    #[ignore]
+    fn test_send_recv() {
+        let manager = PeerManager::init(
+            Network::Bitcoin,
+            ".",
+            None,
+            LogDiscoveryProgress,
+            None,
+            None,
+        )
+        .unwrap();
+
+        let target_address = manager.connected_cbf_addresses().unwrap()[0];
+
+        let ping = NetworkMessage::Ping(30);
+
+        println!("Asking peer {}", target_address);
+
+        manager.send_to(&target_address, ping).unwrap();
+
+        let response = manager
+            .receive_from(&target_address, "pong")
+            .unwrap()
+            .unwrap();
+
+        let value = match response {
+            NetworkMessage::Pong(v) => Some(v),
+            _ => None,
+        };
+
+        let value = value.unwrap();
+
+        println!("Got value {:#?}", value);
+    }
+
+    #[test]
+    #[ignore]
+    fn test_connect_all() {
+        let manager = PeerManager::init(
+            Network::Bitcoin,
+            ".",
+            None,
+            LogDiscoveryProgress,
+            None,
+            None,
+        )
+        .unwrap();
+
+        let cbf_pings = vec![100u64; manager.min_cbf];
+        let non_cbf_pings = vec![200u64; manager.min_total - manager.min_cbf];
+
+        let cbf_peers = manager.connected_cbf_addresses().unwrap();
+        let non_cbf_peers = manager.connected_non_cbf_addresses().unwrap();
+
+        let sent_cbf: Vec<bool> = cbf_pings
+            .iter()
+            .zip(cbf_peers.iter())
+            .map(|(ping, address)| {
+                let message = NetworkMessage::Ping(*ping);
+                manager.send_to(address, message).unwrap();
+                true
+            })
+            .collect();
+
+        assert_eq!(sent_cbf, vec![true; manager.min_cbf]);
+
+        println!("Sent pings to cbf peers");
+
+        let sent_noncbf: Vec<bool> = non_cbf_pings
+            .iter()
+            .zip(non_cbf_peers.iter())
+            .map(|(ping, address)| {
+                let message = NetworkMessage::Ping(*ping);
+                manager.send_to(address, message).unwrap();
+                true
+            })
+            .collect();
+
+        assert_eq!(sent_noncbf, vec![true; manager.min_total - manager.min_cbf]);
+
+        println!("Sent pings to non cbf peers");
+
+        let cbf_received: Vec<u64> = cbf_peers
+            .iter()
+            .map(|address| {
+                let response = manager.receive_from(address, "pong").unwrap().unwrap();
+
+                let value = match response {
+                    NetworkMessage::Pong(v) => Some(v),
+                    _ => None,
+                };
+
+                value.unwrap()
+            })
+            .collect();
+
+        let non_cbf_received: Vec<u64> = non_cbf_peers
+            .iter()
+            .map(|address| {
+                let response = manager.receive_from(address, "pong").unwrap().unwrap();
+
+                let value = match response {
+                    NetworkMessage::Pong(v) => Some(v),
+                    _ => None,
+                };
+
+                value.unwrap()
+            })
+            .collect();
+
+        assert_eq!(cbf_pings, cbf_received);
+
+        assert_eq!(non_cbf_pings, non_cbf_received);
+    }
+}

src/blockchain/electrum.rs

@@ -43,11 +43,17 @@ use crate::FeeRate;
 ///
 /// ## Example
 /// See the [`blockchain::electrum`](crate::blockchain::electrum) module for a usage example.
-pub struct ElectrumBlockchain(Client);
+pub struct ElectrumBlockchain {
+    client: Client,
+    stop_gap: usize,
+}
 
 impl std::convert::From<Client> for ElectrumBlockchain {
     fn from(client: Client) -> Self {
-        ElectrumBlockchain(client)
+        ElectrumBlockchain {
+            client,
+            stop_gap: 20,
+        }
     }
 }
 
@@ -64,34 +70,33 @@ impl Blockchain for ElectrumBlockchain {
 
     fn setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        self.0
-            .electrum_like_setup(stop_gap, database, progress_update)
+        self.client
+            .electrum_like_setup(self.stop_gap, database, progress_update)
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self.0.transaction_get(txid).map(Option::Some)?)
+        Ok(self.client.transaction_get(txid).map(Option::Some)?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self.0.transaction_broadcast(tx).map(|_| ())?)
+        Ok(self.client.transaction_broadcast(tx).map(|_| ())?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
         // TODO: unsubscribe when added to the client, or is there a better call to use here?
 
         Ok(self
-            .0
+            .client
             .block_headers_subscribe()
             .map(|data| data.height as u32)?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
         Ok(FeeRate::from_btc_per_kvb(
-            self.0.estimate_fee(target)? as f32
+            self.client.estimate_fee(target)? as f32
         ))
     }
 }
@@ -149,6 +154,8 @@ pub struct ElectrumBlockchainConfig {
     pub retry: u8,
     /// Request timeout (seconds)
     pub timeout: Option<u8>,
+    /// Stop searching addresses for transactions after finding an unused gap of this length
+    pub stop_gap: usize,
 }
 
 impl ConfigurableBlockchain for ElectrumBlockchain {
@@ -162,16 +169,17 @@ impl ConfigurableBlockchain for ElectrumBlockchain {
             .socks5(socks5)?
             .build();
 
-        Ok(ElectrumBlockchain(Client::from_config(
-            config.url.as_str(),
-            electrum_config,
-        )?))
+        Ok(ElectrumBlockchain {
+            client: Client::from_config(config.url.as_str(), electrum_config)?,
+            stop_gap: config.stop_gap,
+        })
     }
 }
 
-#[cfg(feature = "test-blockchains")]
+#[cfg(test)]
+#[cfg(feature = "test-electrum")]
 crate::bdk_blockchain_tests! {
-    fn test_instance() -> ElectrumBlockchain {
-        ElectrumBlockchain::from(Client::new(&testutils::blockchain_tests::get_electrum_url()).unwrap())
+    fn test_instance(test_client: &TestClient) -> ElectrumBlockchain {
+        ElectrumBlockchain::from(Client::new(&test_client.electrsd.electrum_url).unwrap())
     }
 }

src/blockchain/esplora.rs

@@ -18,34 +18,32 @@
 //!
 //! ```no_run
 //! # use bdk::blockchain::esplora::EsploraBlockchain;
-//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", None);
+//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", None, 20);
 //! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::collections::{HashMap, HashSet};
 use std::fmt;
 
-use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
-
-#[allow(unused_imports)]
-use log::{debug, error, info, trace};
-
-use serde::Deserialize;
-
-use reqwest::{Client, StatusCode};
-
 use bitcoin::consensus::{self, deserialize, serialize};
 use bitcoin::hashes::hex::{FromHex, ToHex};
 use bitcoin::hashes::{sha256, Hash};
 use bitcoin::{BlockHash, BlockHeader, Script, Transaction, Txid};
+use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
+#[allow(unused_imports)]
+use log::{debug, error, info, trace};
+use reqwest::{Client, StatusCode};
+use serde::Deserialize;
 
-use self::utils::{ElectrumLikeSync, ElsGetHistoryRes};
-use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::wallet::utils::ChunksIterator;
 use crate::FeeRate;
 
+use super::*;
+
+use self::utils::{ElectrumLikeSync, ElsGetHistoryRes};
+
 const DEFAULT_CONCURRENT_REQUESTS: u8 = 4;
 
 #[derive(Debug)]
@@ -62,22 +60,31 @@ struct UrlClient {
 /// ## Example
 /// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
 #[derive(Debug)]
-pub struct EsploraBlockchain(UrlClient);
+pub struct EsploraBlockchain {
+    url_client: UrlClient,
+    stop_gap: usize,
+}
 
 impl std::convert::From<UrlClient> for EsploraBlockchain {
     fn from(url_client: UrlClient) -> Self {
-        EsploraBlockchain(url_client)
+        EsploraBlockchain {
+            url_client,
+            stop_gap: 20,
+        }
     }
 }
 
 impl EsploraBlockchain {
     /// Create a new instance of the client from a base URL
-    pub fn new(base_url: &str, concurrency: Option<u8>) -> Self {
-        EsploraBlockchain(UrlClient {
-            url: base_url.to_string(),
-            client: Client::new(),
-            concurrency: concurrency.unwrap_or(DEFAULT_CONCURRENT_REQUESTS),
-        })
+    pub fn new(base_url: &str, concurrency: Option<u8>, stop_gap: usize) -> Self {
+        EsploraBlockchain {
+            url_client: UrlClient {
+                url: base_url.to_string(),
+                client: Client::new(),
+                concurrency: concurrency.unwrap_or(DEFAULT_CONCURRENT_REQUESTS),
+            },
+            stop_gap,
+        }
     }
 }
 
@@ -95,29 +102,28 @@ impl Blockchain for EsploraBlockchain {
 
     fn setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
         maybe_await!(self
-            .0
-            .electrum_like_setup(stop_gap, database, progress_update))
+            .url_client
+            .electrum_like_setup(self.stop_gap, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(await_or_block!(self.0._get_tx(txid))?)
+        Ok(await_or_block!(self.url_client._get_tx(txid))?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(await_or_block!(self.0._broadcast(tx))?)
+        Ok(await_or_block!(self.url_client._broadcast(tx))?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
-        Ok(await_or_block!(self.0._get_height())?)
+        Ok(await_or_block!(self.url_client._get_height())?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
-        let estimates = await_or_block!(self.0._get_fee_estimates())?;
+        let estimates = await_or_block!(self.url_client._get_fee_estimates())?;
 
         let fee_val = estimates
             .into_iter()
@@ -369,6 +375,8 @@ pub struct EsploraBlockchainConfig {
     pub base_url: String,
     /// Number of parallel requests sent to the esplora service (default: 4)
     pub concurrency: Option<u8>,
+    /// Stop searching addresses for transactions after finding an unused gap of this length
+    pub stop_gap: usize,
 }
 
 impl ConfigurableBlockchain for EsploraBlockchain {
@@ -378,6 +386,7 @@ impl ConfigurableBlockchain for EsploraBlockchain {
         Ok(EsploraBlockchain::new(
             config.base_url.as_str(),
             config.concurrency,
+            config.stop_gap,
         ))
     }
 }
@@ -415,9 +424,10 @@ impl_error!(std::num::ParseIntError, Parsing, EsploraError);
 impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
 impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
 
-#[cfg(feature = "test-blockchains")]
+#[cfg(test)]
+#[cfg(feature = "test-esplora")]
 crate::bdk_blockchain_tests! {
-    fn test_instance() -> EsploraBlockchain {
-        EsploraBlockchain::new(std::env::var("BDK_ESPLORA_URL").unwrap_or("127.0.0.1:3002".into()).as_str(), None)
+    fn test_instance(test_client: &TestClient) -> EsploraBlockchain {
+        EsploraBlockchain::new(&format!("http://{}",test_client.electrsd.esplora_url.as_ref().unwrap()), None, 20)
     }
 }

src/blockchain/mod.rs

@@ -43,6 +43,14 @@ pub use self::electrum::ElectrumBlockchain;
 #[cfg(feature = "electrum")]
 pub use self::electrum::ElectrumBlockchainConfig;
 
+#[cfg(feature = "rpc")]
+#[cfg_attr(docsrs, doc(cfg(feature = "rpc")))]
+pub mod rpc;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcBlockchain;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcConfig;
+
 #[cfg(feature = "esplora")]
 #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
 pub mod esplora;
@@ -52,6 +60,7 @@ pub use self::esplora::EsploraBlockchain;
 #[cfg(feature = "compact_filters")]
 #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
 pub mod compact_filters;
+
 #[cfg(feature = "compact_filters")]
 pub use self::compact_filters::CompactFiltersBlockchain;
 
@@ -84,7 +93,6 @@ pub trait Blockchain {
     /// [`Blockchain::sync`] defaults to calling this internally if not overridden.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error>;
@@ -109,11 +117,10 @@ pub trait Blockchain {
     /// [`BatchOperations::del_utxo`]: crate::database::BatchOperations::del_utxo
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.setup(stop_gap, database, progress_update))
+        maybe_await!(self.setup(database, progress_update))
     }
 
     /// Fetch a transaction from the blockchain given its txid
@@ -166,7 +173,7 @@ impl Progress for Sender<ProgressData> {
 }
 
 /// Type that implements [`Progress`] and drops every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct NoopProgress;
 
 /// Create a new instance of [`NoopProgress`]
@@ -181,7 +188,7 @@ impl Progress for NoopProgress {
 }
 
 /// Type that implements [`Progress`] and logs at level `INFO` every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct LogProgress;
 
 /// Create a nwe instance of [`LogProgress`]
@@ -209,20 +216,18 @@ impl<T: Blockchain> Blockchain for Arc<T> {
 
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.deref().setup(stop_gap, database, progress_update))
+        maybe_await!(self.deref().setup(database, progress_update))
     }
 
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.deref().sync(stop_gap, database, progress_update))
+        maybe_await!(self.deref().sync(database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {

src/blockchain/rpc.rs

@@ -0,0 +1,437 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Riccardo Casatta <riccardo@casatta.it>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Rpc Blockchain
+//!
+//! Backend that gets blockchain data from Bitcoin Core RPC
+//!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::{RpcConfig, RpcBlockchain, ConfigurableBlockchain};
+//! let config = RpcConfig {
+//!     url: "127.0.0.1:18332".to_string(),
+//!     auth: bitcoincore_rpc::Auth::CookieFile("/home/user/.bitcoin/.cookie".into()),
+//!     network: bdk::bitcoin::Network::Testnet,
+//!     wallet_name: "wallet_name".to_string(),
+//!     skip_blocks: None,
+//! };
+//! let blockchain = RpcBlockchain::from_config(&config);
+//! ```
+
+use crate::bitcoin::consensus::deserialize;
+use crate::bitcoin::{Address, Network, OutPoint, Transaction, TxOut, Txid};
+use crate::blockchain::{Blockchain, Capability, ConfigurableBlockchain, Progress};
+use crate::database::{BatchDatabase, DatabaseUtils};
+use crate::descriptor::{get_checksum, IntoWalletDescriptor};
+use crate::wallet::utils::SecpCtx;
+use crate::{ConfirmationTime, Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails};
+use bitcoincore_rpc::json::{
+    GetAddressInfoResultLabel, ImportMultiOptions, ImportMultiRequest,
+    ImportMultiRequestScriptPubkey, ImportMultiRescanSince,
+};
+use bitcoincore_rpc::jsonrpc::serde_json::Value;
+use bitcoincore_rpc::{Auth, Client, RpcApi};
+use log::debug;
+use serde::Deserialize;
+use std::collections::{HashMap, HashSet};
+use std::str::FromStr;
+
+/// The main struct for RPC backend implementing the [crate::blockchain::Blockchain] trait
+#[derive(Debug)]
+pub struct RpcBlockchain {
+    /// Rpc client to the node, includes the wallet name
+    client: Client,
+    /// Network used
+    network: Network,
+    /// Blockchain capabilities, cached here at startup
+    capabilities: HashSet<Capability>,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    skip_blocks: Option<u32>,
+
+    /// This is a fixed Address used as a hack key to store information on the node
+    _storage_address: Address,
+}
+
+/// RpcBlockchain configuration options
+#[derive(Debug)]
+pub struct RpcConfig {
+    /// The bitcoin node url
+    pub url: String,
+    /// The bitcoin node authentication mechanism
+    pub auth: Auth,
+    /// The network we are using (it will be checked the bitcoin node network matches this)
+    pub network: Network,
+    /// The wallet name in the bitcoin node, consider using [wallet_name_from_descriptor] for this
+    pub wallet_name: String,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    pub skip_blocks: Option<u32>,
+}
+
+impl RpcBlockchain {
+    fn get_node_synced_height(&self) -> Result<u32, Error> {
+        let info = self.client.get_address_info(&self._storage_address)?;
+        if let Some(GetAddressInfoResultLabel::Simple(label)) = info.labels.first() {
+            Ok(label
+                .parse::<u32>()
+                .unwrap_or_else(|_| self.skip_blocks.unwrap_or(0)))
+        } else {
+            Ok(self.skip_blocks.unwrap_or(0))
+        }
+    }
+
+    /// Set the synced height in the core node by using a label of a fixed address so that
+    /// another client with the same descriptor doesn't rescan the blockchain
+    fn set_node_synced_height(&self, height: u32) -> Result<(), Error> {
+        Ok(self
+            .client
+            .set_label(&self._storage_address, &height.to_string())?)
+    }
+}
+
+impl Blockchain for RpcBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        self.capabilities.clone()
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        let mut scripts_pubkeys = database.iter_script_pubkeys(Some(KeychainKind::External))?;
+        scripts_pubkeys.extend(database.iter_script_pubkeys(Some(KeychainKind::Internal))?);
+        debug!(
+            "importing {} script_pubkeys (some maybe already imported)",
+            scripts_pubkeys.len()
+        );
+        let requests: Vec<_> = scripts_pubkeys
+            .iter()
+            .map(|s| ImportMultiRequest {
+                timestamp: ImportMultiRescanSince::Timestamp(0),
+                script_pubkey: Some(ImportMultiRequestScriptPubkey::Script(&s)),
+                watchonly: Some(true),
+                ..Default::default()
+            })
+            .collect();
+        let options = ImportMultiOptions {
+            rescan: Some(false),
+        };
+        // Note we use import_multi because as of bitcoin core 0.21.0 many descriptors are not supported
+        // https://bitcoindevkit.org/descriptors/#compatibility-matrix
+        //TODO maybe convenient using import_descriptor for compatible descriptor and import_multi as fallback
+        self.client.import_multi(&requests, Some(&options))?;
+
+        let current_height = self.get_height()?;
+
+        // min because block invalidate may cause height to go down
+        let node_synced = self.get_node_synced_height()?.min(current_height);
+
+        //TODO call rescan in chunks (updating node_synced_height) so that in case of
+        // interruption work can be partially recovered
+        debug!(
+            "rescan_blockchain from:{} to:{}",
+            node_synced, current_height
+        );
+        self.client
+            .rescan_blockchain(Some(node_synced as usize), Some(current_height as usize))?;
+        progress_update.update(1.0, None)?;
+
+        self.set_node_synced_height(current_height)?;
+
+        self.sync(database, progress_update)
+    }
+
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        db: &mut D,
+        _progress_update: P,
+    ) -> Result<(), Error> {
+        let mut indexes = HashMap::new();
+        for keykind in &[KeychainKind::External, KeychainKind::Internal] {
+            indexes.insert(*keykind, db.get_last_index(*keykind)?.unwrap_or(0));
+        }
+
+        let mut known_txs: HashMap<_, _> = db
+            .iter_txs(true)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let known_utxos: HashSet<_> = db.iter_utxos()?.into_iter().collect();
+
+        //TODO list_since_blocks would be more efficient
+        let current_utxo = self
+            .client
+            .list_unspent(Some(0), None, None, Some(true), None)?;
+        debug!("current_utxo len {}", current_utxo.len());
+
+        //TODO supported up to 1_000 txs, should use since_blocks or do paging
+        let list_txs = self
+            .client
+            .list_transactions(None, Some(1_000), None, Some(true))?;
+        let mut list_txs_ids = HashSet::new();
+
+        for tx_result in list_txs.iter().filter(|t| {
+            // list_txs returns all conflicting tx we want to
+            // filter out replaced tx => unconfirmed and not in the mempool
+            t.info.confirmations > 0 || self.client.get_mempool_entry(&t.info.txid).is_ok()
+        }) {
+            let txid = tx_result.info.txid;
+            list_txs_ids.insert(txid);
+            if let Some(mut known_tx) = known_txs.get_mut(&txid) {
+                let confirmation_time =
+                    ConfirmationTime::new(tx_result.info.blockheight, tx_result.info.blocktime);
+                if confirmation_time != known_tx.confirmation_time {
+                    // reorg may change tx height
+                    debug!(
+                        "updating tx({}) confirmation time to: {:?}",
+                        txid, confirmation_time
+                    );
+                    known_tx.confirmation_time = confirmation_time;
+                    db.set_tx(&known_tx)?;
+                }
+            } else {
+                //TODO check there is already the raw tx in db?
+                let tx_result = self.client.get_transaction(&txid, Some(true))?;
+                let tx: Transaction = deserialize(&tx_result.hex)?;
+                let mut received = 0u64;
+                let mut sent = 0u64;
+                for output in tx.output.iter() {
+                    if let Ok(Some((kind, index))) =
+                        db.get_path_from_script_pubkey(&output.script_pubkey)
+                    {
+                        if index > *indexes.get(&kind).unwrap() {
+                            indexes.insert(kind, index);
+                        }
+                        received += output.value;
+                    }
+                }
+
+                for input in tx.input.iter() {
+                    if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+                        sent += previous_output.value;
+                    }
+                }
+
+                let td = TransactionDetails {
+                    transaction: Some(tx),
+                    txid: tx_result.info.txid,
+                    confirmation_time: ConfirmationTime::new(
+                        tx_result.info.blockheight,
+                        tx_result.info.blocktime,
+                    ),
+                    received,
+                    sent,
+                    fee: tx_result.fee.map(|f| f.as_sat().abs() as u64),
+                    verified: true,
+                };
+                debug!(
+                    "saving tx: {} tx_result.fee:{:?} td.fees:{:?}",
+                    td.txid, tx_result.fee, td.fee
+                );
+                db.set_tx(&td)?;
+            }
+        }
+
+        for known_txid in known_txs.keys() {
+            if !list_txs_ids.contains(known_txid) {
+                debug!("removing tx: {}", known_txid);
+                db.del_tx(known_txid, false)?;
+            }
+        }
+
+        let current_utxos: HashSet<_> = current_utxo
+            .into_iter()
+            .map(|u| {
+                Ok(LocalUtxo {
+                    outpoint: OutPoint::new(u.txid, u.vout),
+                    keychain: db
+                        .get_path_from_script_pubkey(&u.script_pub_key)?
+                        .ok_or(Error::TransactionNotFound)?
+                        .0,
+                    txout: TxOut {
+                        value: u.amount.as_sat(),
+                        script_pubkey: u.script_pub_key,
+                    },
+                })
+            })
+            .collect::<Result<_, Error>>()?;
+
+        let spent: HashSet<_> = known_utxos.difference(&current_utxos).collect();
+        for s in spent {
+            debug!("removing utxo: {:?}", s);
+            db.del_utxo(&s.outpoint)?;
+        }
+        let received: HashSet<_> = current_utxos.difference(&known_utxos).collect();
+        for s in received {
+            debug!("adding utxo: {:?}", s);
+            db.set_utxo(s)?;
+        }
+
+        for (keykind, index) in indexes {
+            debug!("{:?} max {}", keykind, index);
+            db.set_last_index(keykind, index)?;
+        }
+
+        Ok(())
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(Some(self.client.get_raw_transaction(txid, None)?))
+    }
+
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        Ok(self.client.send_raw_transaction(tx).map(|_| ())?)
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(self.client.get_blockchain_info().map(|i| i.blocks as u32)?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        let sat_per_kb = self
+            .client
+            .estimate_smart_fee(target as u16, None)?
+            .fee_rate
+            .ok_or(Error::FeeRateUnavailable)?
+            .as_sat() as f64;
+
+        Ok(FeeRate::from_sat_per_vb((sat_per_kb / 1000f64) as f32))
+    }
+}
+
+impl ConfigurableBlockchain for RpcBlockchain {
+    type Config = RpcConfig;
+
+    /// Returns RpcBlockchain backend creating an RPC client to a specific wallet named as the descriptor's checksum
+    /// if it's the first time it creates the wallet in the node and upon return is granted the wallet is loaded
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let wallet_name = config.wallet_name.clone();
+        let wallet_url = format!("{}/wallet/{}", config.url, &wallet_name);
+        debug!("connecting to {} auth:{:?}", wallet_url, config.auth);
+
+        let client = Client::new(wallet_url, config.auth.clone())?;
+        let loaded_wallets = client.list_wallets()?;
+        if loaded_wallets.contains(&wallet_name) {
+            debug!("wallet already loaded {:?}", wallet_name);
+        } else {
+            let existing_wallets = list_wallet_dir(&client)?;
+            if existing_wallets.contains(&wallet_name) {
+                client.load_wallet(&wallet_name)?;
+                debug!("wallet loaded {:?}", wallet_name);
+            } else {
+                client.create_wallet(&wallet_name, Some(true), None, None, None)?;
+                debug!("wallet created {:?}", wallet_name);
+            }
+        }
+
+        let blockchain_info = client.get_blockchain_info()?;
+        let network = match blockchain_info.chain.as_str() {
+            "main" => Network::Bitcoin,
+            "test" => Network::Testnet,
+            "regtest" => Network::Regtest,
+            "signet" => Network::Signet,
+            _ => return Err(Error::Generic("Invalid network".to_string())),
+        };
+        if network != config.network {
+            return Err(Error::InvalidNetwork {
+                requested: config.network,
+                found: network,
+            });
+        }
+
+        let mut capabilities: HashSet<_> = vec![Capability::FullHistory].into_iter().collect();
+        let rpc_version = client.version()?;
+        if rpc_version >= 210_000 {
+            let info: HashMap<String, Value> = client.call("getindexinfo", &[]).unwrap();
+            if info.contains_key("txindex") {
+                capabilities.insert(Capability::GetAnyTx);
+                capabilities.insert(Capability::AccurateFees);
+            }
+        }
+
+        // this is just a fixed address used only to store a label containing the synced height in the node
+        let mut storage_address =
+            Address::from_str("bc1qst0rewf0wm4kw6qn6kv0e5tc56nkf9yhcxlhqv").unwrap();
+        storage_address.network = network;
+
+        Ok(RpcBlockchain {
+            client,
+            network,
+            capabilities,
+            _storage_address: storage_address,
+            skip_blocks: config.skip_blocks,
+        })
+    }
+}
+
+/// Deterministically generate a unique name given the descriptors defining the wallet
+pub fn wallet_name_from_descriptor<T>(
+    descriptor: T,
+    change_descriptor: Option<T>,
+    network: Network,
+    secp: &SecpCtx,
+) -> Result<String, Error>
+where
+    T: IntoWalletDescriptor,
+{
+    //TODO check descriptors contains only public keys
+    let descriptor = descriptor
+        .into_wallet_descriptor(&secp, network)?
+        .0
+        .to_string();
+    let mut wallet_name = get_checksum(&descriptor[..descriptor.find('#').unwrap()])?;
+    if let Some(change_descriptor) = change_descriptor {
+        let change_descriptor = change_descriptor
+            .into_wallet_descriptor(&secp, network)?
+            .0
+            .to_string();
+        wallet_name.push_str(
+            get_checksum(&change_descriptor[..change_descriptor.find('#').unwrap()])?.as_str(),
+        );
+    }
+
+    Ok(wallet_name)
+}
+
+/// return the wallets available in default wallet directory
+//TODO use bitcoincore_rpc method when PR #179 lands
+fn list_wallet_dir(client: &Client) -> Result<Vec<String>, Error> {
+    #[derive(Deserialize)]
+    struct Name {
+        name: String,
+    }
+    #[derive(Deserialize)]
+    struct CallResult {
+        wallets: Vec<Name>,
+    }
+
+    let result: CallResult = client.call("listwalletdir", &[])?;
+    Ok(result.wallets.into_iter().map(|n| n.name).collect())
+}
+
+#[cfg(test)]
+#[cfg(feature = "test-rpc")]
+crate::bdk_blockchain_tests! {
+
+    fn test_instance(test_client: &TestClient) -> RpcBlockchain {
+        let config = RpcConfig {
+            url: test_client.bitcoind.rpc_url(),
+            auth: Auth::CookieFile(test_client.bitcoind.params.cookie_file.clone()),
+            network: Network::Regtest,
+            wallet_name: format!("client-wallet-test-{:?}", std::time::SystemTime::now() ),
+            skip_blocks: None,
+        };
+        RpcBlockchain::from_config(&config).unwrap()
+    }
+}

src/blockchain/utils.rs

@@ -21,7 +21,7 @@ use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{KeychainKind, LocalUtxo, TransactionDetails};
+use crate::types::{ConfirmationTime, KeychainKind, LocalUtxo, TransactionDetails};
 use crate::wallet::time::Instant;
 use crate::wallet::utils::ChunksIterator;
 
@@ -53,7 +53,7 @@ pub trait ElectrumLikeSync {
 
     fn electrum_like_setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
+        stop_gap: usize,
         db: &mut D,
         _progress_update: P,
     ) -> Result<(), Error> {
@@ -61,7 +61,6 @@ pub trait ElectrumLikeSync {
         let start = Instant::new();
         debug!("start setup");
 
-        let stop_gap = stop_gap.unwrap_or(20);
         let chunk_size = stop_gap;
 
         let mut history_txs_id = HashSet::new();
@@ -147,18 +146,19 @@ pub trait ElectrumLikeSync {
         // save any tx details not in db but in history_txs_id or with different height/timestamp
         for txid in history_txs_id.iter() {
             let height = txid_height.get(txid).cloned().flatten();
-            let timestamp = *new_timestamps.get(txid).unwrap_or(&0u64);
+            let timestamp = new_timestamps.get(txid).cloned();
             if let Some(tx_details) = txs_details_in_db.get(txid) {
-                // check if height matches, otherwise updates it
-                if tx_details.height != height {
+                // check if tx height matches, otherwise updates it. timestamp is not in the if clause
+                // because we are not asking headers for confirmed tx we know about
+                if tx_details.confirmation_time.as_ref().map(|c| c.height) != height {
+                    let confirmation_time = ConfirmationTime::new(height, timestamp);
                     let mut new_tx_details = tx_details.clone();
-                    new_tx_details.height = height;
-                    new_tx_details.timestamp = timestamp;
+                    new_tx_details.confirmation_time = confirmation_time;
                     batch.set_tx(&new_tx_details)?;
                 }
             } else {
                 save_transaction_details_and_utxos(
-                    &txid,
+                    txid,
                     db,
                     timestamp,
                     height,
@@ -171,7 +171,7 @@ pub trait ElectrumLikeSync {
         // remove any tx details in db but not in history_txs_id
         for txid in txs_details_in_db.keys() {
             if !history_txs_id.contains(txid) {
-                batch.del_tx(&txid, false)?;
+                batch.del_tx(txid, false)?;
             }
         }
 
@@ -238,9 +238,13 @@ pub trait ElectrumLikeSync {
         chunk_size: usize,
     ) -> Result<HashMap<Txid, u64>, Error> {
         let mut txid_timestamp = HashMap::new();
+        let txid_in_db_with_conf: HashSet<_> = txs_details_in_db
+            .values()
+            .filter_map(|details| details.confirmation_time.as_ref().map(|_| details.txid))
+            .collect();
         let needed_txid_height: HashMap<&Txid, u32> = txid_height
             .iter()
-            .filter(|(t, _)| txs_details_in_db.get(*t).is_none())
+            .filter(|(t, _)| !txid_in_db_with_conf.contains(*t))
             .filter_map(|(t, o)| o.map(|h| (t, h)))
             .collect();
         let needed_heights: HashSet<u32> = needed_txid_height.values().cloned().collect();
@@ -292,7 +296,7 @@ pub trait ElectrumLikeSync {
 fn save_transaction_details_and_utxos<D: BatchDatabase>(
     txid: &Txid,
     db: &mut D,
-    timestamp: u64,
+    timestamp: Option<u64>,
     height: Option<u32>,
     updates: &mut dyn BatchOperations,
     utxo_deps: &HashMap<OutPoint, OutPoint>,
@@ -329,7 +333,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
 
         // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
         if let Some(outpoint) = utxo_deps.get(&input.previous_output) {
-            updates.del_utxo(&outpoint)?;
+            updates.del_utxo(outpoint)?;
         }
     }
 
@@ -355,9 +359,9 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
         transaction: Some(tx),
         received: incoming,
         sent: outgoing,
-        height,
-        timestamp,
-        fees: inputs_sum.saturating_sub(outputs_sum), /* if the tx is a coinbase, fees would be negative */
+        confirmation_time: ConfirmationTime::new(height, timestamp),
+        fee: Some(inputs_sum.saturating_sub(outputs_sum)), /* if the tx is a coinbase, fees would be negative */
+        verified: height.is_some(),
     };
     updates.set_tx(&tx_details)?;
 

src/database/keyvalue.rs

@@ -320,7 +320,7 @@ impl Database for Tree {
             .map(|b| -> Result<_, Error> {
                 let mut txdetails: TransactionDetails = serde_json::from_slice(&b)?;
                 if include_raw {
-                    txdetails.transaction = self.get_raw_tx(&txid)?;
+                    txdetails.transaction = self.get_raw_tx(txid)?;
                 }
 
                 Ok(txdetails)
@@ -383,6 +383,7 @@ impl BatchDatabase for Tree {
 
 #[cfg(test)]
 mod test {
+    use lazy_static::lazy_static;
     use std::sync::{Arc, Condvar, Mutex, Once};
     use std::time::{SystemTime, UNIX_EPOCH};
 

src/database/memory.rs

@@ -394,7 +394,7 @@ impl Database for MemoryDatabase {
         Ok(self.map.get(&key).map(|b| {
             let mut txdetails: TransactionDetails = b.downcast_ref().cloned().unwrap();
             if include_raw {
-                txdetails.transaction = self.get_raw_tx(&txid).unwrap();
+                txdetails.transaction = self.get_raw_tx(txid).unwrap();
             }
 
             txdetails
@@ -429,8 +429,8 @@ impl BatchDatabase for MemoryDatabase {
     }
 
     fn commit_batch(&mut self, mut batch: Self::Batch) -> Result<(), Error> {
-        for key in batch.deleted_keys {
-            self.map.remove(&key);
+        for key in batch.deleted_keys.iter() {
+            self.map.remove(key);
         }
         self.map.append(&mut batch.map);
         Ok(())
@@ -473,18 +473,19 @@ macro_rules! populate_test_db {
         };
 
         let txid = tx.txid();
-        let height = tx_meta
-            .min_confirmations
-            .map(|conf| current_height.unwrap().checked_sub(conf as u32).unwrap());
+        let confirmation_time = tx_meta.min_confirmations.map(|conf| ConfirmationTime {
+            height: current_height.unwrap().checked_sub(conf as u32).unwrap(),
+            timestamp: 0,
+        });
 
         let tx_details = TransactionDetails {
             transaction: Some(tx.clone()),
             txid,
-            timestamp: 0,
-            height,
+            fee: Some(0),
             received: 0,
             sent: 0,
-            fees: 0,
+            confirmation_time,
+            verified: current_height.is_some(),
         };
 
         db.set_tx(&tx_details).unwrap();

src/database/mod.rs

@@ -314,11 +314,14 @@ pub mod test {
         let mut tx_details = TransactionDetails {
             transaction: Some(tx),
             txid,
-            timestamp: 123456,
             received: 1337,
             sent: 420420,
-            fees: 140,
-            height: Some(1000),
+            fee: Some(140),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 123456,
+                height: 1000,
+            }),
+            verified: true,
         };
 
         tree.set_tx(&tx_details).unwrap();

src/descriptor/dsl.rs

@@ -175,7 +175,7 @@ macro_rules! impl_node_opcode_two {
 #[doc(hidden)]
 #[macro_export]
 macro_rules! impl_node_opcode_three {
-    ( $terminal_variant:ident, $( $inner:tt )* ) => {
+    ( $terminal_variant:ident, $( $inner:tt )* ) => ({
         use $crate::descriptor::CheckMiniscript;
 
         let inner = $crate::fragment_internal!( @t $( $inner )* );
@@ -201,7 +201,7 @@ macro_rules! impl_node_opcode_three {
 
                 Ok((minisc, a_keymap, networks))
             })
-    };
+    });
 }
 
 #[doc(hidden)]
@@ -790,6 +790,25 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_fixed_threeop_descriptors() {
+        let redeem_key = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        let move_key = bitcoin::PublicKey::from_str(
+            "032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af",
+        )
+        .unwrap();
+
+        check(
+            descriptor!(sh(wsh(and_or(pk(redeem_key), older(1000), pk(move_key))))),
+            true,
+            true,
+            &["2MypGwr5eQWAWWJtiJgUEToVxc4zuokjQRe"],
+        );
+    }
+
     #[test]
     fn test_bip32_legacy_descriptors() {
         let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();

src/descriptor/mod.rs

@@ -128,11 +128,11 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
             let (pk, _, networks) = if self.0.is_witness() {
                 let desciptor_key: DescriptorKey<miniscript::Segwitv0> =
                     pk.clone().into_descriptor_key()?;
-                desciptor_key.extract(&secp)?
+                desciptor_key.extract(secp)?
             } else {
                 let desciptor_key: DescriptorKey<miniscript::Legacy> =
                     pk.clone().into_descriptor_key()?;
-                desciptor_key.extract(&secp)?
+                desciptor_key.extract(secp)?
             };
 
             if networks.contains(&network) {

src/descriptor/policy.rs

@@ -336,7 +336,7 @@ impl Satisfaction {
                         items.push(inner_index);
                         let conditions_set = other_conditions
                             .values()
-                            .fold(HashSet::new(), |set, i| set.union(&i).cloned().collect());
+                            .fold(HashSet::new(), |set, i| set.union(i).cloned().collect());
                         conditions.insert(inner_index, conditions_set);
                     }
                 }
@@ -1007,7 +1007,6 @@ mod test {
     use crate::descriptor::{ExtractPolicy, IntoWalletDescriptor};
 
     use super::*;
-    use crate::bitcoin::consensus::deserialize;
     use crate::descriptor::derived::AsDerived;
     use crate::descriptor::policy::SatisfiableItem::{Multisig, Signature, Thresh};
     use crate::keys::{DescriptorKey, IntoDescriptorKey};
@@ -1031,8 +1030,8 @@ mod test {
     ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
         let path = bip32::DerivationPath::from_str(path).unwrap();
         let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
-        let tpub = bip32::ExtendedPubKey::from_private(&secp, &tprv);
-        let fingerprint = tprv.fingerprint(&secp);
+        let tpub = bip32::ExtendedPubKey::from_private(secp, &tprv);
+        let fingerprint = tprv.fingerprint(secp);
         let prvkey = (tprv, path.clone()).into_descriptor_key().unwrap();
         let pubkey = (tpub, path).into_descriptor_key().unwrap();
 
@@ -1475,7 +1474,7 @@ mod test {
 
         let signers_container = Arc::new(SignersContainer::from(keymap));
 
-        let psbt: Psbt = deserialize(&base64::decode(ALICE_SIGNED_PSBT).unwrap()).unwrap();
+        let psbt = Psbt::from_str(ALICE_SIGNED_PSBT).unwrap();
 
         let policy_alice_psbt = wallet_desc
             .extract_policy(&signers_container, BuildSatisfaction::Psbt(&psbt), &secp)
@@ -1490,7 +1489,7 @@ mod test {
             )
         );
 
-        let psbt: Psbt = deserialize(&base64::decode(BOB_SIGNED_PSBT).unwrap()).unwrap();
+        let psbt = Psbt::from_str(BOB_SIGNED_PSBT).unwrap();
         let policy_bob_psbt = wallet_desc
             .extract_policy(&signers_container, BuildSatisfaction::Psbt(&psbt), &secp)
             .unwrap()
@@ -1504,7 +1503,7 @@ mod test {
             )
         );
 
-        let psbt: Psbt = deserialize(&base64::decode(ALICE_BOB_SIGNED_PSBT).unwrap()).unwrap();
+        let psbt = Psbt::from_str(ALICE_BOB_SIGNED_PSBT).unwrap();
         let policy_alice_bob_psbt = wallet_desc
             .extract_policy(&signers_container, BuildSatisfaction::Psbt(&psbt), &secp)
             .unwrap()
@@ -1545,8 +1544,7 @@ mod test {
             addr.to_string()
         );
 
-        let psbt: Psbt =
-            deserialize(&base64::decode(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED).unwrap()).unwrap();
+        let psbt = Psbt::from_str(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED).unwrap();
 
         let build_sat = BuildSatisfaction::PsbtTimelocks {
             psbt: &psbt,
@@ -1584,9 +1582,7 @@ mod test {
         );
         //println!("{}", serde_json::to_string(&policy_expired).unwrap());
 
-        let psbt_signed: Psbt =
-            deserialize(&base64::decode(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED_SIGNED).unwrap())
-                .unwrap();
+        let psbt_signed = Psbt::from_str(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED_SIGNED).unwrap();
 
         let build_sat_expired_signed = BuildSatisfaction::PsbtTimelocks {
             psbt: &psbt_signed,

src/error.rs

@@ -11,6 +11,7 @@
 
 use std::fmt;
 
+use crate::bitcoin::Network;
 use crate::{descriptor, wallet, wallet::address_validator};
 use bitcoin::OutPoint;
 
@@ -23,10 +24,6 @@ pub enum Error {
     Generic(String),
     /// This error is thrown when trying to convert Bare and Public key script to address
     ScriptDoesntHaveAddressForm,
-    /// Found multiple outputs when `single_recipient` option has been specified
-    SingleRecipientMultipleOutputs,
-    /// `single_recipient` option is selected but neither `drain_wallet` nor `manually_selected_only` are
-    SingleRecipientNoInputs,
     /// Cannot build a tx without recipients
     NoRecipients,
     /// `manually_selected_only` option is selected but no utxo has been passed
@@ -64,6 +61,8 @@ pub enum Error {
         /// Required fee absolute value (satoshi)
         required: u64,
     },
+    /// Node doesn't have data to estimate a fee rate
+    FeeRateUnavailable,
     /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
     /// key in the descriptor must either be a master key itself (having depth = 0) or have an
     /// explicit origin provided
@@ -80,6 +79,16 @@ pub enum Error {
     InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
     /// Signing error
     Signer(crate::wallet::signer::SignerError),
+    /// Invalid network
+    InvalidNetwork {
+        /// requested network, for example what is given as bdk-cli option
+        requested: Network,
+        /// found network, for example the network of the bitcoin node
+        found: Network,
+    },
+    #[cfg(feature = "verify")]
+    /// Transaction verification error
+    Verification(crate::wallet::verify::VerifyError),
 
     /// Progress value must be between `0.0` (included) and `100.0` (included)
     InvalidProgressValue(f32),
@@ -106,6 +115,8 @@ pub enum Error {
     Hex(bitcoin::hashes::hex::Error),
     /// Partially signed bitcoin transaction error
     Psbt(bitcoin::util::psbt::Error),
+    /// Partially signed bitcoin transaction parseerror
+    PsbtParse(bitcoin::util::psbt::PsbtParseError),
 
     //KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
     //MissingInputUTXO(usize),
@@ -126,6 +137,9 @@ pub enum Error {
     #[cfg(feature = "key-value-db")]
     /// Sled database error
     Sled(sled::Error),
+    #[cfg(feature = "rpc")]
+    /// Rpc client error
+    Rpc(bitcoincore_rpc::Error),
 }
 
 impl fmt::Display for Error {
@@ -172,6 +186,7 @@ impl_error!(bitcoin::secp256k1::Error, Secp256k1);
 impl_error!(serde_json::Error, Json);
 impl_error!(bitcoin::hashes::hex::Error, Hex);
 impl_error!(bitcoin::util::psbt::Error, Psbt);
+impl_error!(bitcoin::util::psbt::PsbtParseError, PsbtParse);
 
 #[cfg(feature = "electrum")]
 impl_error!(electrum_client::Error, Electrum);
@@ -179,6 +194,8 @@ impl_error!(electrum_client::Error, Electrum);
 impl_error!(crate::blockchain::esplora::EsploraError, Esplora);
 #[cfg(feature = "key-value-db")]
 impl_error!(sled::Error, Sled);
+#[cfg(feature = "rpc")]
+impl_error!(bitcoincore_rpc::Error, Rpc);
 
 #[cfg(feature = "compact_filters")]
 impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
@@ -189,3 +206,13 @@ impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
         }
     }
 }
+
+#[cfg(feature = "verify")]
+impl From<crate::wallet::verify::VerifyError> for Error {
+    fn from(other: crate::wallet::verify::VerifyError) -> Self {
+        match other {
+            crate::wallet::verify::VerifyError::Global(inner) => *inner,
+            err => Error::Verification(err),
+        }
+    }
+}

src/lib.rs

@@ -40,7 +40,7 @@
 //! interact with the bitcoin P2P network.
 //!
 //! ```toml
-//! bdk = "0.8.0"
+//! bdk = "0.9.0"
 //! ```
 #![cfg_attr(
     feature = "electrum",
@@ -104,8 +104,6 @@ fn main() -> Result<(), bdk::Error> {
 
 ### Example
 ```no_run
-use base64::decode;
-
 use bdk::{FeeRate, Wallet};
 use bdk::database::MemoryDatabase;
 use bdk::blockchain::{noop_progress, ElectrumBlockchain};
@@ -138,7 +136,7 @@ fn main() -> Result<(), bdk::Error> {
     };
 
     println!("Transaction details: {:#?}", details);
-    println!("Unsigned PSBT: {}", base64::encode(&serialize(&psbt)));
+    println!("Unsigned PSBT: {}", &psbt);
 
     Ok(())
 }
@@ -150,8 +148,9 @@ fn main() -> Result<(), bdk::Error> {
 //!
 //! ### Example
 //! ```no_run
-//! use base64::decode;
-//! use bitcoin::consensus::deserialize;
+//! use std::str::FromStr;
+//!
+//! use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
 //!
 //! use bdk::{Wallet, SignOptions};
 //! use bdk::database::MemoryDatabase;
@@ -165,7 +164,7 @@ fn main() -> Result<(), bdk::Error> {
 //!     )?;
 //!
 //!     let psbt = "...";
-//!     let mut psbt = deserialize(&base64::decode(psbt).unwrap())?;
+//!     let mut psbt = Psbt::from_str(psbt)?;
 //!
 //!     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
 //!
@@ -223,6 +222,9 @@ extern crate bdk_macros;
 #[cfg(feature = "compact_filters")]
 extern crate lazy_static;
 
+#[cfg(feature = "rpc")]
+pub extern crate bitcoincore_rpc;
+
 #[cfg(feature = "electrum")]
 pub extern crate electrum_client;
 
@@ -233,12 +235,6 @@ pub extern crate reqwest;
 pub extern crate sled;
 
 #[allow(unused_imports)]
-#[cfg(test)]
-#[allow(unused_imports)]
-#[cfg(test)]
-#[macro_use]
-pub extern crate serial_test;
-
 #[macro_use]
 pub(crate) mod error;
 pub mod blockchain;

src/psbt/mod.rs

@@ -41,12 +41,12 @@ impl PsbtUtils for Psbt {
 
 #[cfg(test)]
 mod test {
-    use crate::bitcoin::consensus::deserialize;
     use crate::bitcoin::TxIn;
     use crate::psbt::Psbt;
     use crate::wallet::test::{get_funded_wallet, get_test_wpkh};
     use crate::wallet::AddressIndex;
     use crate::SignOptions;
+    use std::str::FromStr;
 
     // from bip 174
     const PSBT_STR: &str = "cHNidP8BAKACAAAAAqsJSaCMWvfEm4IS9Bfi8Vqz9cM9zxU4IagTn4d6W3vkAAAAAAD+////qwlJoIxa98SbghL0F+LxWrP1wz3PFTghqBOfh3pbe+QBAAAAAP7///8CYDvqCwAAAAAZdqkUdopAu9dAy+gdmI5x3ipNXHE5ax2IrI4kAAAAAAAAGXapFG9GILVT+glechue4O/p+gOcykWXiKwAAAAAAAEHakcwRAIgR1lmF5fAGwNrJZKJSGhiGDR9iYZLcZ4ff89X0eURZYcCIFMJ6r9Wqk2Ikf/REf3xM286KdqGbX+EhtdVRs7tr5MZASEDXNxh/HupccC1AaZGoqg7ECy0OIEhfKaC3Ibi1z+ogpIAAQEgAOH1BQAAAAAXqRQ1RebjO4MsRwUPJNPuuTycA5SLx4cBBBYAFIXRNTfy4mVAWjTbr6nj3aAfuCMIAAAA";
@@ -54,7 +54,7 @@ mod test {
     #[test]
     #[should_panic(expected = "InputIndexOutOfRange")]
     fn test_psbt_malformed_psbt_input_legacy() {
-        let psbt_bip: Psbt = deserialize(&base64::decode(PSBT_STR).unwrap()).unwrap();
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let send_to = wallet.get_address(AddressIndex::New).unwrap();
         let mut builder = wallet.build_tx();
@@ -71,7 +71,7 @@ mod test {
     #[test]
     #[should_panic(expected = "InputIndexOutOfRange")]
     fn test_psbt_malformed_psbt_input_segwit() {
-        let psbt_bip: Psbt = deserialize(&base64::decode(PSBT_STR).unwrap()).unwrap();
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let send_to = wallet.get_address(AddressIndex::New).unwrap();
         let mut builder = wallet.build_tx();
@@ -103,7 +103,7 @@ mod test {
 
     #[test]
     fn test_psbt_sign_with_finalized() {
-        let psbt_bip: Psbt = deserialize(&base64::decode(PSBT_STR).unwrap()).unwrap();
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let send_to = wallet.get_address(AddressIndex::New).unwrap();
         let mut builder = wallet.build_tx();

src/testutils/blockchain_tests.rs

@@ -6,38 +6,49 @@ use bitcoin::{Address, Amount, Script, Transaction, Txid};
 pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
 pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
 use core::str::FromStr;
+use electrsd::bitcoind::BitcoinD;
+use electrsd::{bitcoind, ElectrsD};
 pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 use std::collections::HashMap;
 use std::env;
 use std::ops::Deref;
-use std::path::PathBuf;
 use std::time::Duration;
 
 pub struct TestClient {
-    client: RpcClient,
-    electrum: ElectrumClient,
+    pub bitcoind: BitcoinD,
+    pub electrsd: ElectrsD,
 }
 
 impl TestClient {
-    pub fn new(rpc_host_and_wallet: String, rpc_wallet_name: String) -> Self {
-        let client = RpcClient::new(
-            format!("http://{}/wallet/{}", rpc_host_and_wallet, rpc_wallet_name),
-            get_auth(),
-        )
-        .unwrap();
-        let electrum = ElectrumClient::new(&get_electrum_url()).unwrap();
+    pub fn new(bitcoind_exe: String, electrs_exe: String) -> Self {
+        debug!("launching {} and {}", &bitcoind_exe, &electrs_exe);
+        let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
 
-        TestClient { client, electrum }
+        let http_enabled = cfg!(feature = "test-esplora");
+
+        let electrsd = ElectrsD::new(electrs_exe, &bitcoind, false, http_enabled).unwrap();
+
+        let node_address = bitcoind.client.get_new_address(None, None).unwrap();
+        bitcoind
+            .client
+            .generate_to_address(101, &node_address)
+            .unwrap();
+
+        let mut test_client = TestClient { bitcoind, electrsd };
+        TestClient::wait_for_block(&mut test_client, 101);
+        test_client
     }
 
     fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
         // wait for electrs to index the tx
         exponential_backoff_poll(|| {
+            self.electrsd.trigger().unwrap();
             trace!("wait_for_tx {}", txid);
 
-            self.electrum
+            self.electrsd
+                .client
                 .script_get_history(monitor_script)
                 .unwrap()
                 .iter()
@@ -46,12 +57,13 @@ impl TestClient {
     }
 
     fn wait_for_block(&mut self, min_height: usize) {
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         loop {
             let header = exponential_backoff_poll(|| {
-                self.electrum.ping().unwrap();
-                self.electrum.block_headers_pop().unwrap()
+                self.electrsd.trigger().unwrap();
+                self.electrsd.client.ping().unwrap();
+                self.electrsd.client.block_headers_pop().unwrap()
             });
             if header.height >= min_height {
                 break;
@@ -96,10 +108,13 @@ impl TestClient {
             .unwrap();
 
         // broadcast through electrum so that it caches the tx immediately
+
         let txid = self
-            .electrum
+            .electrsd
+            .client
             .transaction_broadcast(&deserialize(&tx.hex).unwrap())
             .unwrap();
+        debug!("broadcasted to electrum {}", txid);
 
         if let Some(num) = meta_tx.min_confirmations {
             self.generate(num, None);
@@ -209,7 +224,7 @@ impl TestClient {
         let block_hex: String = serialize(&block).to_hex();
         debug!("generated block hex: {}", block_hex);
 
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         let submit_result: serde_json::Value =
             self.call("submitblock", &[block_hex.into()]).unwrap();
@@ -237,7 +252,7 @@ impl TestClient {
     }
 
     pub fn invalidate(&mut self, num_blocks: u64) {
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         let best_hash = self.get_best_block_hash().unwrap();
         let initial_height = self.get_block_info(&best_hash).unwrap().height;
@@ -288,16 +303,25 @@ impl Deref for TestClient {
     type Target = RpcClient;
 
     fn deref(&self) -> &Self::Target {
-        &self.client
+        &self.bitcoind.client
     }
 }
 
 impl Default for TestClient {
     fn default() -> Self {
-        let rpc_host_and_port =
-            env::var("BDK_RPC_URL").unwrap_or_else(|_| "127.0.0.1:18443".to_string());
-        let wallet = env::var("BDK_RPC_WALLET").unwrap_or_else(|_| "bdk-test".to_string());
-        Self::new(rpc_host_and_port, wallet)
+        let bitcoind_exe = env::var("BITCOIND_EXE")
+            .ok()
+            .or(bitcoind::downloaded_exe_path())
+            .expect(
+                "you should provide env var BITCOIND_EXE or specifiy a bitcoind version feature",
+            );
+        let electrs_exe = env::var("ELECTRS_EXE")
+            .ok()
+            .or(electrsd::downloaded_exe_path())
+            .expect(
+                "you should provide env var ELECTRS_EXE or specifiy a electrsd version feature",
+            );
+        Self::new(bitcoind_exe, electrs_exe)
     }
 }
 
@@ -317,27 +341,13 @@ where
     }
 }
 
-// TODO: we currently only support env vars, we could also parse a toml file
-fn get_auth() -> Auth {
-    match env::var("BDK_RPC_AUTH").as_ref().map(String::as_ref) {
-        Ok("USER_PASS") => Auth::UserPass(
-            env::var("BDK_RPC_USER").unwrap(),
-            env::var("BDK_RPC_PASS").unwrap(),
-        ),
-        _ => Auth::CookieFile(PathBuf::from(
-            env::var("BDK_RPC_COOKIEFILE")
-                .unwrap_or_else(|_| "/home/user/.bitcoin/regtest/.cookie".to_string()),
-        )),
-    }
-}
-
 /// This macro runs blockchain tests against a `Blockchain` implementation. It requires access to a
 /// Bitcoin core wallet via RPC. At the moment you have to dig into the code yourself and look at
 /// the setup required to run the tests yourself.
 #[macro_export]
 macro_rules! bdk_blockchain_tests {
     (
-     fn test_instance() -> $blockchain:ty $block:block) => {
+     fn $_fn_name:ident ( $( $test_client:ident : &TestClient )? $(,)? ) -> $blockchain:ty $block:block) => {
         #[cfg(test)]
         mod bdk_blockchain_tests {
             use $crate::bitcoin::Network;
@@ -346,18 +356,18 @@ macro_rules! bdk_blockchain_tests {
             use $crate::database::MemoryDatabase;
             use $crate::types::KeychainKind;
             use $crate::{Wallet, FeeRate};
-            use $crate::wallet::AddressIndex::New;
             use $crate::testutils;
-            use $crate::serial_test::serial;
 
             use super::*;
 
-            fn get_blockchain() -> $blockchain {
+            #[allow(unused_variables)]
+            fn get_blockchain(test_client: &TestClient) -> $blockchain {
+                $( let $test_client = test_client; )?
                 $block
             }
 
-            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<$blockchain, MemoryDatabase> {
-                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
+            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>), test_client: &TestClient) -> Wallet<$blockchain, MemoryDatabase> {
+                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain(test_client)).unwrap()
             }
 
             fn init_single_sig() -> (Wallet<$blockchain, MemoryDatabase>, (String, Option<String>), TestClient) {
@@ -368,13 +378,16 @@ macro_rules! bdk_blockchain_tests {
                 };
 
                 let test_client = TestClient::default();
-                let wallet = get_wallet_from_descriptors(&descriptors);
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
+
+                // rpc need to call import_multi before receiving any tx, otherwise will not see tx in the mempool
+                #[cfg(feature = "test-rpc")]
+                wallet.sync(noop_progress(), None).unwrap();
 
                 (wallet, descriptors, test_client)
             }
 
             #[test]
-            #[serial]
             fn test_sync_simple() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -386,18 +399,17 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External, "incorrect keychain kind");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.received, 50_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time");
             }
 
             #[test]
-            #[serial]
             fn test_sync_stop_gap_20() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -410,12 +422,11 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 100_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                assert_eq!(wallet.get_balance().unwrap(), 100_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
             }
 
             #[test]
-            #[serial]
             fn test_sync_before_and_after_receive() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -428,12 +439,11 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
             }
 
             #[test]
-            #[serial]
             fn test_sync_multiple_outputs_same_tx() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -443,19 +453,18 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 105_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 3);
+                assert_eq!(wallet.get_balance().unwrap(), 105_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 3, "incorrect number of unspents");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.received, 105_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 105_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_multi() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -468,13 +477,12 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 2);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 2, "incorrect number of unspent");
             }
 
             #[test]
-            #[serial]
             fn test_sync_address_reuse() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -490,11 +498,10 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_rbf_replaced() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -504,36 +511,35 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.received, 50_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
 
                 let new_txid = test_client.bump_fee(&txid);
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after bump");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs after bump");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent after bump");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, new_txid);
-                assert_eq!(list_tx_item.received, 50_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, new_txid, "incorrect txid after bump");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received after bump");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent after bump");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect height after bump");
             }
 
             // FIXME: I would like this to be cfg_attr(not(feature = "test-esplora"), ignore) but it
             // doesn't work for some reason.
             #[cfg(not(feature = "esplora"))]
             #[test]
-            #[serial]
             fn test_sync_reorg_block() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -543,28 +549,27 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert!(list_tx_item.height.is_some());
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert!(list_tx_item.confirmation_time.is_some(), "incorrect confirmation_time");
 
                 // Invalidate 1 block
                 test_client.invalidate(1);
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after invalidate");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid after invalidate");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time after invalidate");
             }
 
             #[test]
-            #[serial]
             fn test_sync_after_send() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 println!("{}", descriptors.0);
@@ -575,7 +580,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey(), 25_000);
@@ -585,18 +590,17 @@ macro_rules! bdk_blockchain_tests {
                 let tx = psbt.extract_tx();
                 println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
                 wallet.broadcast(tx).unwrap();
-
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after send");
 
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
             }
 
             #[test]
-            #[serial]
-            fn test_sync_outgoing_from_scratch() {
+            fn test_update_confirmation_time_after_generate() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
+                println!("{}", descriptors.0);
                 let node_addr = test_client.get_node_address(None);
 
                 let received_txid = test_client.receive(testutils! {
@@ -604,36 +608,63 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_none());
+
+                test_client.generate(1, Some(node_addr));
+                wallet.sync(noop_progress(), None).unwrap();
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_some());
+
+            }
+
+            #[test]
+            fn test_sync_outgoing_from_scratch() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+                let received_txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey(), 25_000);
                 let (mut psbt, details) = builder.finish().unwrap();
+
                 let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
                 assert!(finalized, "Cannot finalize transaction");
                 let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after receive");
 
                 // empty wallet
-                let wallet = get_wallet_from_descriptors(&descriptors);
-                wallet.sync(noop_progress(), None).unwrap();
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
 
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
+                wallet.sync(noop_progress(), None).unwrap();
                 let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
 
                 let received = tx_map.get(&received_txid).unwrap();
-                assert_eq!(received.received, 50_000);
-                assert_eq!(received.sent, 0);
+                assert_eq!(received.received, 50_000, "incorrect received from receiver");
+                assert_eq!(received.sent, 0, "incorrect sent from receiver");
 
                 let sent = tx_map.get(&sent_txid).unwrap();
-                assert_eq!(sent.received, details.received);
-                assert_eq!(sent.sent, details.sent);
-                assert_eq!(sent.fees, details.fees);
+                assert_eq!(sent.received, details.received, "incorrect received from sender");
+                assert_eq!(sent.sent, details.sent, "incorrect sent from sender");
+                assert_eq!(sent.fee.unwrap_or(0), details.fee.unwrap_or(0), "incorrect fees from sender");
             }
 
             #[test]
-            #[serial]
             fn test_sync_long_change_chain() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -643,7 +674,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut total_sent = 0;
                 for _ in 0..5 {
@@ -656,21 +687,26 @@ macro_rules! bdk_blockchain_tests {
 
                     wallet.sync(noop_progress(), None).unwrap();
 
-                    total_sent += 5_000 + details.fees;
+                    total_sent += 5_000 + details.fee.unwrap_or(0);
                 }
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance after chain");
 
                 // empty wallet
-                let wallet = get_wallet_from_descriptors(&descriptors);
+
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
+
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance empty wallet");
+
             }
 
             #[test]
-            #[serial]
-            fn test_sync_bump_fee() {
+            fn test_sync_bump_fee_basic() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
 
@@ -679,7 +715,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 5_000).enable_rbf();
@@ -688,8 +724,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fees - 5_000);
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance from received");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(2.1));
@@ -698,14 +734,13 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fees - 5_000);
-                assert_eq!(wallet.get_balance().unwrap(), new_details.received);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees after bump");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance from received after bump");
 
-                assert!(new_details.fees > details.fees);
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_remove_change() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -715,7 +750,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
@@ -724,8 +759,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fees);
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect received after send");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(5.0));
@@ -734,15 +769,14 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 0);
-                assert_eq!(new_details.received, 0);
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after change removal");
+                assert_eq!(new_details.received, 0, "incorrect received after change removal");
 
-                assert!(new_details.fees > details.fees);
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
             }
 
             #[test]
-            #[serial]
-            fn test_sync_bump_fee_add_input() {
+            fn test_sync_bump_fee_add_input_simple() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
 
@@ -751,7 +785,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
@@ -760,8 +794,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                assert_eq!(details.received, 1_000 - details.fees);
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(10.0));
@@ -770,12 +804,11 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(new_details.sent, 75_000);
-                assert_eq!(wallet.get_balance().unwrap(), new_details.received);
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance after add input");
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_add_input_no_change() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -785,7 +818,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
@@ -794,8 +827,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                assert_eq!(details.received, 1_000 - details.fees);
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(123.0));
@@ -806,25 +839,38 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(new_details.sent, 75_000);
-                assert_eq!(wallet.get_balance().unwrap(), 0);
-                assert_eq!(new_details.received, 0);
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after add input");
+                assert_eq!(new_details.received, 0, "incorrect received after add input");
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_coinbase() {
                 let (wallet, _, mut test_client) = init_single_sig();
-                let wallet_addr = wallet.get_address(New).unwrap().address;
+
+                let wallet_addr = wallet.get_address($crate::wallet::AddressIndex::New).unwrap().address;
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 0);
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance");
 
                 test_client.generate(1, Some(wallet_addr));
 
+                #[cfg(feature = "rpc")]
+                {
+                    // rpc consider coinbase only when mature (100 blocks)
+                    let node_addr = test_client.get_node_address(None);
+                    test_client.generate(100, Some(node_addr));
+                }
+
+
                 wallet.sync(noop_progress(), None).unwrap();
-                assert!(wallet.get_balance().unwrap() > 0);
+                assert!(wallet.get_balance().unwrap() > 0, "incorrect balance after receiving coinbase");
             }
         }
-    }
+    };
+
+    ( fn $fn_name:ident ($( $tt:tt )+) -> $blockchain:ty $block:block) => {
+        compile_error!(concat!("Invalid arguments `", stringify!($($tt)*), "` in the blockchain tests fn."));
+        compile_error!("Only the exact `&TestClient` type is supported, **without** any leading path items.");
+    };
 }

src/testutils/mod.rs

@@ -10,6 +10,7 @@
 // licenses.
 #![allow(missing_docs)]
 
+#[cfg(test)]
 #[cfg(feature = "test-blockchains")]
 pub mod blockchain_tests;
 

src/types.rs

@@ -80,7 +80,7 @@ impl std::default::Default for FeeRate {
 /// An unspent output owned by a [`Wallet`].
 ///
 /// [`Wallet`]: crate::Wallet
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
 pub struct LocalUtxo {
     /// Reference to a transaction output
     pub outpoint: OutPoint,
@@ -139,7 +139,7 @@ impl Utxo {
                 }
 
                 if let Some(txout) = &psbt_input.witness_utxo {
-                    return &txout;
+                    return txout;
                 }
 
                 unreachable!("Foreign UTXOs will always have one of these set")
@@ -155,16 +155,47 @@ pub struct TransactionDetails {
     pub transaction: Option<Transaction>,
     /// Transaction id
     pub txid: Txid,
-    /// Timestamp
-    pub timestamp: u64,
+
     /// Received value (sats)
     pub received: u64,
     /// Sent value (sats)
     pub sent: u64,
-    /// Fee value (sats)
-    pub fees: u64,
-    /// Confirmed in block height, `None` means unconfirmed
-    pub height: Option<u32>,
+    /// Fee value (sats) if available.
+    /// The availability of the fee depends on the backend. It's never `None` with an Electrum
+    /// Server backend, but it could be `None` with a Bitcoin RPC node without txindex that receive
+    /// funds while offline.
+    pub fee: Option<u64>,
+    /// If the transaction is confirmed, contains height and timestamp of the block containing the
+    /// transaction, unconfirmed transaction contains `None`.
+    pub confirmation_time: Option<ConfirmationTime>,
+    /// Whether the tx has been verified against the consensus rules
+    ///
+    /// Confirmed txs are considered "verified" by default, while unconfirmed txs are checked to
+    /// ensure an unstrusted [`Blockchain`](crate::blockchain::Blockchain) backend can't trick the
+    /// wallet into using an invalid tx as an RBF template.
+    ///
+    /// The check is only perfomed when the `verify` feature is enabled.
+    #[serde(default = "bool::default")] // default to `false` if not specified
+    pub verified: bool,
+}
+
+/// Block height and timestamp of the block containing the confirmed transaction
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
+pub struct ConfirmationTime {
+    /// confirmation block height
+    pub height: u32,
+    /// confirmation block timestamp
+    pub timestamp: u64,
+}
+
+impl ConfirmationTime {
+    /// Returns `Some` `ConfirmationTime` if both `height` and `timestamp` are `Some`
+    pub fn new(height: Option<u32>, timestamp: Option<u64>) -> Option<Self> {
+        match (height, timestamp) {
+            (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+            _ => None,
+        }
+    }
 }
 
 #[cfg(test)]

src/wallet/coin_selection.rs

@@ -90,6 +90,7 @@
 //! ```
 
 use crate::types::FeeRate;
+use crate::wallet::Vbytes;
 use crate::{database::Database, WeightedUtxo};
 use crate::{error::Error, Utxo};
 
@@ -257,8 +258,8 @@ struct OutputGroup {
 
 impl OutputGroup {
     fn new(weighted_utxo: WeightedUtxo, fee_rate: FeeRate) -> Self {
-        let fee = (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as f32 / 4.0
-            * fee_rate.as_sat_vb();
+        let fee =
+            (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight).vbytes() * fee_rate.as_sat_vb();
         let effective_value = weighted_utxo.utxo.txout().value as i64 - fee.ceil() as i64;
         OutputGroup {
             weighted_utxo,
@@ -862,7 +863,7 @@ mod test {
 
         assert_eq!(result.selected.len(), 1);
         assert_eq!(result.selected_amount(), 100_000);
-        let input_size = (TXIN_BASE_WEIGHT as f32) / 4.0 + P2WPKH_WITNESS_SIZE as f32 / 4.0;
+        let input_size = (TXIN_BASE_WEIGHT + P2WPKH_WITNESS_SIZE).vbytes();
         let epsilon = 0.5;
         assert!((1.0 - (result.fee_amount / input_size)).abs() < epsilon);
     }

src/wallet/export.rs

@@ -128,7 +128,7 @@ impl WalletExport {
             Ok(txs) => {
                 let mut heights = txs
                     .into_iter()
-                    .map(|tx| tx.height.unwrap_or(0))
+                    .map(|tx| tx.confirmation_time.map(|c| c.height).unwrap_or(0))
                     .collect::<Vec<_>>();
                 heights.sort_unstable();
 
@@ -212,6 +212,7 @@ mod test {
     use crate::database::{memory::MemoryDatabase, BatchOperations};
     use crate::types::TransactionDetails;
     use crate::wallet::Wallet;
+    use crate::ConfirmationTime;
 
     fn get_test_db() -> MemoryDatabase {
         let mut db = MemoryDatabase::new();
@@ -221,11 +222,15 @@ mod test {
                 "4ddff1fa33af17f377f62b72357b43107c19110a8009b36fb832af505efed98a",
             )
             .unwrap(),
-            timestamp: 12345678,
+
             received: 100_000,
             sent: 0,
-            fees: 500,
-            height: Some(5000),
+            fee: Some(500),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 12345678,
+                height: 5000,
+            }),
+            verified: true,
         })
         .unwrap();
 

src/wallet/mod.rs

@@ -42,6 +42,9 @@ pub mod signer;
 pub mod time;
 pub mod tx_builder;
 pub(crate) mod utils;
+#[cfg(feature = "verify")]
+#[cfg_attr(docsrs, doc(cfg(feature = "verify")))]
+pub mod verify;
 
 pub use utils::IsDust;
 
@@ -277,7 +280,7 @@ where
         self.descriptor
             .as_derived(index, &self.secp)
             .address(self.network)
-            .map(|address| AddressInfo { address, index })
+            .map(|address| AddressInfo { index, address })
             .map_err(|_| Error::ScriptDoesntHaveAddressForm)
     }
 
@@ -289,7 +292,7 @@ where
         self.descriptor
             .as_derived(index, &self.secp)
             .address(self.network)
-            .map(|address| AddressInfo { address, index })
+            .map(|address| AddressInfo { index, address })
             .map_err(|_| Error::ScriptDoesntHaveAddressForm)
     }
 
@@ -398,7 +401,7 @@ where
     /// [`TxBuilder`]: crate::TxBuilder
     pub fn build_tx(&self) -> TxBuilder<'_, B, D, DefaultCoinSelectionAlgorithm, CreateTx> {
         TxBuilder {
-            wallet: &self,
+            wallet: self,
             params: TxParams::default(),
             coin_selection: DefaultCoinSelectionAlgorithm::default(),
             phantom: core::marker::PhantomData,
@@ -562,21 +565,6 @@ where
             output: vec![],
         };
 
-        let recipients = match &params.single_recipient {
-            Some(recipient) => vec![(recipient, 0)],
-            None => params.recipients.iter().map(|(r, v)| (r, *v)).collect(),
-        };
-        if params.single_recipient.is_some()
-            && !params.manually_selected_only
-            && !params.drain_wallet
-            && params.bumping_fee.is_none()
-        {
-            return Err(Error::SingleRecipientNoInputs);
-        }
-        if recipients.is_empty() {
-            return Err(Error::NoRecipients);
-        }
-
         if params.manually_selected_only && params.utxos.is_empty() {
             return Err(Error::NoUtxosSelected);
         }
@@ -588,12 +576,12 @@ where
         let calc_fee_bytes = |wu| (wu as f32) * fee_rate.as_sat_vb() / 4.0;
         fee_amount += calc_fee_bytes(tx.get_weight());
 
-        for (index, (script_pubkey, satoshi)) in recipients.into_iter().enumerate() {
-            let value = match params.single_recipient {
-                Some(_) => 0,
-                None if satoshi.is_dust() => return Err(Error::OutputBelowDustLimit(index)),
-                None => satoshi,
-            };
+        let recipients = params.recipients.iter().map(|(r, v)| (r, *v));
+
+        for (index, (script_pubkey, value)) in recipients.enumerate() {
+            if value.is_dust() {
+                return Err(Error::OutputBelowDustLimit(index));
+            }
 
             if self.is_mine(script_pubkey)? {
                 received += value;
@@ -648,54 +636,47 @@ where
             })
             .collect();
 
-        // prepare the change output
-        let change_output = match params.single_recipient {
-            Some(_) => None,
-            None => {
-                let change_script = self.get_change_address()?;
-                let change_output = TxOut {
-                    script_pubkey: change_script,
-                    value: 0,
-                };
+        // prepare the drain output
+        let mut drain_output = {
+            let script_pubkey = match params.drain_to {
+                Some(ref drain_recipient) => drain_recipient.clone(),
+                None => self.get_change_address()?,
+            };
 
-                // take the change into account for fees
-                fee_amount += calc_fee_bytes(serialize(&change_output).len() * 4);
-                Some(change_output)
+            TxOut {
+                script_pubkey,
+                value: 0,
             }
         };
+
+        fee_amount += calc_fee_bytes(serialize(&drain_output).len() * 4);
+
         let mut fee_amount = fee_amount.ceil() as u64;
-        let change_val = (coin_selection.selected_amount() - outgoing).saturating_sub(fee_amount);
+        let drain_val = (coin_selection.selected_amount() - outgoing).saturating_sub(fee_amount);
 
-        match change_output {
-            None if change_val.is_dust() => {
-                // single recipient, but the only output would be below dust limit
-                // TODO: or OutputBelowDustLimit?
-                return Err(Error::InsufficientFunds {
-                    needed: DUST_LIMIT_SATOSHI,
-                    available: change_val,
-                });
-            }
-            Some(_) if change_val.is_dust() => {
-                // skip the change output because it's dust -- just include it in the fee.
-                fee_amount += change_val;
-            }
-            Some(mut change_output) => {
-                change_output.value = change_val;
-                received += change_val;
-
-                tx.output.push(change_output);
-            }
-            None => {
-                // there's only one output, send everything to it
-                tx.output[0].value = change_val;
-
-                // the single recipient is our address
-                if self.is_mine(&tx.output[0].script_pubkey)? {
-                    received = change_val;
+        if tx.output.is_empty() {
+            if params.drain_to.is_some() {
+                if drain_val.is_dust() {
+                    return Err(Error::InsufficientFunds {
+                        needed: DUST_LIMIT_SATOSHI,
+                        available: drain_val,
+                    });
                 }
+            } else {
+                return Err(Error::NoRecipients);
             }
         }
 
+        if drain_val.is_dust() {
+            fee_amount += drain_val;
+        } else {
+            drain_output.value = drain_val;
+            if self.is_mine(&drain_output.script_pubkey)? {
+                received += drain_val;
+            }
+            tx.output.push(drain_output);
+        }
+
         // sort input/outputs according to the chosen algorithm
         params.ordering.sort_tx(&mut tx);
 
@@ -706,11 +687,11 @@ where
         let transaction_details = TransactionDetails {
             transaction: None,
             txid,
-            timestamp: time::get_timestamp(),
+            confirmation_time: None,
             received,
             sent,
-            fees: fee_amount,
-            height: None,
+            fee: Some(fee_amount),
+            verified: true,
         };
 
         Ok((psbt, transaction_details))
@@ -722,10 +703,6 @@ where
     /// *repalce by fee* (RBF). If the transaction can be fee bumped then it returns a [`TxBuilder`]
     /// pre-populated with the inputs and outputs of the original transaction.
     ///
-    /// **NOTE**: if the original transaction was made with [`TxBuilder::set_single_recipient`],
-    /// the [`TxBuilder::maintain_single_recipient`] flag should be enabled to correctly reduce the
-    /// only output's value in order to increase the fees.
-    ///
     /// ## Example
     ///
     /// ```no_run
@@ -769,7 +746,7 @@ where
         let mut details = match self.database.borrow().get_tx(&txid, true)? {
             None => return Err(Error::TransactionNotFound),
             Some(tx) if tx.transaction.is_none() => return Err(Error::TransactionNotFound),
-            Some(tx) if tx.height.is_some() => return Err(Error::TransactionConfirmed),
+            Some(tx) if tx.confirmation_time.is_some() => return Err(Error::TransactionConfirmed),
             Some(tx) => tx,
         };
         let mut tx = details.transaction.take().unwrap();
@@ -777,8 +754,8 @@ where
             return Err(Error::IrreplaceableTransaction);
         }
 
-        let vbytes = tx.get_weight() as f32 / 4.0;
-        let feerate = details.fees as f32 / vbytes;
+        let vbytes = tx.get_weight().vbytes();
+        let feerate = details.fee.ok_or(Error::FeeRateUnavailable)? as f32 / vbytes;
 
         // remove the inputs from the tx and process them
         let original_txin = tx.input.drain(..).collect::<Vec<_>>();
@@ -854,14 +831,14 @@ where
                 .collect(),
             utxos: original_utxos,
             bumping_fee: Some(tx_builder::PreviousFee {
-                absolute: details.fees,
+                absolute: details.fee.ok_or(Error::FeeRateUnavailable)?,
                 rate: feerate,
             }),
             ..Default::default()
         };
 
         Ok(TxBuilder {
-            wallet: &self,
+            wallet: self,
             params,
             coin_selection: DefaultCoinSelectionAlgorithm::default(),
             phantom: core::marker::PhantomData,
@@ -993,7 +970,7 @@ where
                 .database
                 .borrow()
                 .get_tx(&input.previous_output.txid, false)?
-                .map(|tx| tx.height.unwrap_or(std::u32::MAX));
+                .map(|tx| tx.confirmation_time.map(|c| c.height).unwrap_or(u32::MAX));
             let current_height = sign_options.assume_height.or(self.current_height);
 
             debug!(
@@ -1031,7 +1008,7 @@ where
                     match desc.satisfy(
                         &mut tmp_input,
                         (
-                            PsbtInputSatisfier::new(&psbt, n),
+                            PsbtInputSatisfier::new(psbt, n),
                             After::new(current_height, false),
                             Older::new(current_height, create_height, false),
                         ),
@@ -1231,7 +1208,7 @@ where
 
         let satisfies_confirmed = match must_only_use_confirmed_tx {
             true => {
-                let database = self.database.borrow_mut();
+                let database = self.database.borrow();
                 may_spend
                     .iter()
                     .map(|u| {
@@ -1239,7 +1216,7 @@ where
                             .get_tx(&u.0.outpoint.txid, true)
                             .map(|tx| match tx {
                                 None => false,
-                                Some(tx) => tx.height.is_some(),
+                                Some(tx) => tx.confirmation_time.is_some(),
                             })
                     })
                     .collect::<Result<Vec<_>, _>>()?
@@ -1489,12 +1466,14 @@ where
             false => 0,
             true => max_address_param.unwrap_or(CACHE_ADDR_BATCH_SIZE),
         };
+        debug!("max_address {}", max_address);
         if self
             .database
             .borrow()
             .get_script_pubkey_from_path(KeychainKind::External, max_address.saturating_sub(1))?
             .is_none()
         {
+            debug!("caching external addresses");
             run_setup = true;
             self.cache_addresses(KeychainKind::External, 0, max_address)?;
         }
@@ -1511,26 +1490,43 @@ where
                 .get_script_pubkey_from_path(KeychainKind::Internal, max_address.saturating_sub(1))?
                 .is_none()
             {
+                debug!("caching internal addresses");
                 run_setup = true;
                 self.cache_addresses(KeychainKind::Internal, 0, max_address)?;
             }
         }
 
+        debug!("run_setup: {}", run_setup);
         // TODO: what if i generate an address first and cache some addresses?
         // TODO: we should sync if generating an address triggers a new batch to be stored
         if run_setup {
-            maybe_await!(self.client.setup(
-                None,
-                self.database.borrow_mut().deref_mut(),
-                progress_update,
-            ))
+            maybe_await!(self
+                .client
+                .setup(self.database.borrow_mut().deref_mut(), progress_update,))?;
         } else {
-            maybe_await!(self.client.sync(
-                None,
-                self.database.borrow_mut().deref_mut(),
-                progress_update,
-            ))
+            maybe_await!(self
+                .client
+                .sync(self.database.borrow_mut().deref_mut(), progress_update,))?;
         }
+
+        #[cfg(feature = "verify")]
+        {
+            debug!("Verifying transactions...");
+            for mut tx in self.database.borrow().iter_txs(true)? {
+                if !tx.verified {
+                    verify::verify_tx(
+                        tx.transaction.as_ref().ok_or(Error::TransactionNotFound)?,
+                        self.database.borrow().deref(),
+                        &self.client,
+                    )?;
+
+                    tx.verified = true;
+                    self.database.borrow_mut().set_tx(&tx)?;
+                }
+            }
+        }
+
+        Ok(())
     }
 
     /// Return a reference to the internal blockchain client
@@ -1552,6 +1548,18 @@ where
     }
 }
 
+/// Trait implemented by types that can be used to measure weight units.
+pub trait Vbytes {
+    /// Convert weight units to virtual bytes.
+    fn vbytes(self) -> f32;
+}
+
+impl Vbytes for usize {
+    fn vbytes(self) -> f32 {
+        self as f32 / 4.0
+    }
+}
+
 #[cfg(test)]
 pub(crate) mod test {
     use std::str::FromStr;
@@ -1738,7 +1746,7 @@ pub(crate) mod test {
                 dust_change = true;
             )*
 
-            let tx_fee_rate = $fees as f32 / (tx.get_weight() as f32 / 4.0);
+            let tx_fee_rate = $fees as f32 / (tx.get_weight().vbytes());
             let fee_rate = $fee_rate.as_sat_vb();
 
             if !dust_change {
@@ -1970,22 +1978,47 @@ pub(crate) mod test {
     }
 
     #[test]
-    fn test_create_tx_single_recipient_drain_wallet() {
+    fn test_create_tx_drain_wallet_and_drain_to() {
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, details) = builder.finish().unwrap();
 
         assert_eq!(psbt.global.unsigned_tx.output.len(), 1);
         assert_eq!(
             psbt.global.unsigned_tx.output[0].value,
-            50_000 - details.fees
+            50_000 - details.fee.unwrap_or(0)
         );
     }
 
+    #[test]
+    fn test_create_tx_drain_wallet_and_drain_to_and_with_recipient() {
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let addr = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+        let drain_addr = wallet.get_address(New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder
+            .add_recipient(addr.script_pubkey(), 20_000)
+            .drain_to(drain_addr.script_pubkey())
+            .drain_wallet();
+        let (psbt, details) = builder.finish().unwrap();
+        dbg!(&psbt);
+        let outputs = psbt.global.unsigned_tx.output;
+
+        assert_eq!(outputs.len(), 2);
+        let main_output = outputs
+            .iter()
+            .find(|x| x.script_pubkey == addr.script_pubkey())
+            .unwrap();
+        let drain_output = outputs
+            .iter()
+            .find(|x| x.script_pubkey == drain_addr.script_pubkey())
+            .unwrap();
+        assert_eq!(main_output.value, 20_000,);
+        assert_eq!(drain_output.value, 30_000 - details.fee.unwrap_or(0));
+    }
+
     #[test]
     fn test_create_tx_default_fee_rate() {
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
@@ -1994,7 +2027,7 @@ pub(crate) mod test {
         builder.add_recipient(addr.script_pubkey(), 25_000);
         let (psbt, details) = builder.finish().unwrap();
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::default(), @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::default(), @add_signature);
     }
 
     #[test]
@@ -2007,7 +2040,7 @@ pub(crate) mod test {
             .fee_rate(FeeRate::from_sat_per_vb(5.0));
         let (psbt, details) = builder.finish().unwrap();
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::from_sat_per_vb(5.0), @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::from_sat_per_vb(5.0), @add_signature);
     }
 
     #[test]
@@ -2016,16 +2049,16 @@ pub(crate) mod test {
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .drain_wallet()
             .fee_absolute(100);
         let (psbt, details) = builder.finish().unwrap();
 
-        assert_eq!(details.fees, 100);
+        assert_eq!(details.fee.unwrap_or(0), 100);
         assert_eq!(psbt.global.unsigned_tx.output.len(), 1);
         assert_eq!(
             psbt.global.unsigned_tx.output[0].value,
-            50_000 - details.fees
+            50_000 - details.fee.unwrap_or(0)
         );
     }
 
@@ -2035,16 +2068,16 @@ pub(crate) mod test {
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .drain_wallet()
             .fee_absolute(0);
         let (psbt, details) = builder.finish().unwrap();
 
-        assert_eq!(details.fees, 0);
+        assert_eq!(details.fee.unwrap_or(0), 0);
         assert_eq!(psbt.global.unsigned_tx.output.len(), 1);
         assert_eq!(
             psbt.global.unsigned_tx.output[0].value,
-            50_000 - details.fees
+            50_000 - details.fee.unwrap_or(0)
         );
     }
 
@@ -2055,7 +2088,7 @@ pub(crate) mod test {
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .drain_wallet()
             .fee_absolute(60_000);
         let (_psbt, _details) = builder.finish().unwrap();
@@ -2077,7 +2110,7 @@ pub(crate) mod test {
         assert_eq!(psbt.global.unsigned_tx.output[0].value, 25_000);
         assert_eq!(
             psbt.global.unsigned_tx.output[1].value,
-            25_000 - details.fees
+            25_000 - details.fee.unwrap_or(0)
         );
     }
 
@@ -2091,18 +2124,18 @@ pub(crate) mod test {
 
         assert_eq!(psbt.global.unsigned_tx.output.len(), 1);
         assert_eq!(psbt.global.unsigned_tx.output[0].value, 49_800);
-        assert_eq!(details.fees, 200);
+        assert_eq!(details.fee.unwrap_or(0), 200);
     }
 
     #[test]
     #[should_panic(expected = "InsufficientFunds")]
-    fn test_create_tx_single_recipient_dust_amount() {
+    fn test_create_tx_drain_to_dust_amount() {
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let addr = wallet.get_address(New).unwrap();
         // very high fee rate, so that the only output would be below dust
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .drain_wallet()
             .fee_rate(FeeRate::from_sat_per_vb(453.0));
         builder.finish().unwrap();
@@ -2122,7 +2155,7 @@ pub(crate) mod test {
         assert_eq!(psbt.global.unsigned_tx.output.len(), 3);
         assert_eq!(
             psbt.global.unsigned_tx.output[0].value,
-            10_000 - details.fees
+            10_000 - details.fee.unwrap_or(0)
         );
         assert_eq!(psbt.global.unsigned_tx.output[1].value, 10_000);
         assert_eq!(psbt.global.unsigned_tx.output[2].value, 30_000);
@@ -2163,9 +2196,7 @@ pub(crate) mod test {
         let (wallet, _, _) = get_funded_wallet("wpkh([d34db33f/44'/0'/0']tpubDEnoLuPdBep9bzw5LoGYpsxUQYheRQ9gcgrJhJEcdKFB9cWQRyYmkCyRoTqeD4tJYiVVgt6A3rN6rWn9RYhR9sBsGxji29LYWHuKKbdb1ev/0/*)");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         assert_eq!(psbt.inputs[0].bip32_derivation.len(), 1);
@@ -2189,9 +2220,7 @@ pub(crate) mod test {
 
         let addr = testutils!(@external descriptors, 5);
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         assert_eq!(psbt.outputs[0].bip32_derivation.len(), 1);
@@ -2212,9 +2241,7 @@ pub(crate) mod test {
             get_funded_wallet("sh(pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW))");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         assert_eq!(
@@ -2237,9 +2264,7 @@ pub(crate) mod test {
             get_funded_wallet("wsh(pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW))");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         assert_eq!(psbt.inputs[0].redeem_script, None);
@@ -2262,9 +2287,7 @@ pub(crate) mod test {
             get_funded_wallet("sh(wsh(pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)))");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         let script = Script::from(
@@ -2284,9 +2307,7 @@ pub(crate) mod test {
             get_funded_wallet("sh(pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW))");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         assert!(psbt.inputs[0].non_witness_utxo.is_some());
@@ -2300,7 +2321,7 @@ pub(crate) mod test {
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .only_witness_utxo()
             .drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
@@ -2315,9 +2336,7 @@ pub(crate) mod test {
             get_funded_wallet("sh(wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW))");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         assert!(psbt.inputs[0].witness_utxo.is_some());
@@ -2329,9 +2348,7 @@ pub(crate) mod test {
             get_funded_wallet("wsh(pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW))");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (psbt, _) = builder.finish().unwrap();
 
         assert!(psbt.inputs[0].non_witness_utxo.is_some());
@@ -2492,7 +2509,7 @@ pub(crate) mod test {
 
         assert_eq!(
             details.sent - details.received,
-            10_000 + details.fees,
+            10_000 + details.fee.unwrap_or(0),
             "we should have only net spent ~10_000"
         );
 
@@ -2758,7 +2775,10 @@ pub(crate) mod test {
         let txid = tx.txid();
         // skip saving the utxos, we know they can't be used anyways
         details.transaction = Some(tx);
-        details.height = Some(42);
+        details.confirmation_time = Some(ConfirmationTime {
+            timestamp: 12345678,
+            height: 42,
+        });
         wallet.database.borrow_mut().set_tx(&details).unwrap();
 
         wallet.build_fee_bump(txid).unwrap().finish().unwrap();
@@ -2863,10 +2883,10 @@ pub(crate) mod test {
 
         assert_eq!(details.sent, original_details.sent);
         assert_eq!(
-            details.received + details.fees,
-            original_details.received + original_details.fees
+            details.received + details.fee.unwrap_or(0),
+            original_details.received + original_details.fee.unwrap_or(0)
         );
-        assert!(details.fees > original_details.fees);
+        assert!(details.fee.unwrap_or(0) > original_details.fee.unwrap_or(0));
 
         let tx = &psbt.global.unsigned_tx;
         assert_eq!(tx.output.len(), 2);
@@ -2887,7 +2907,7 @@ pub(crate) mod test {
             details.received
         );
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::from_sat_per_vb(2.5), @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::from_sat_per_vb(2.5), @add_signature);
     }
 
     #[test]
@@ -2924,14 +2944,14 @@ pub(crate) mod test {
 
         assert_eq!(details.sent, original_details.sent);
         assert_eq!(
-            details.received + details.fees,
-            original_details.received + original_details.fees
+            details.received + details.fee.unwrap_or(0),
+            original_details.received + original_details.fee.unwrap_or(0)
         );
         assert!(
-            details.fees > original_details.fees,
+            details.fee.unwrap_or(0) > original_details.fee.unwrap_or(0),
             "{} > {}",
-            details.fees,
-            original_details.fees
+            details.fee.unwrap_or(0),
+            original_details.fee.unwrap_or(0)
         );
 
         let tx = &psbt.global.unsigned_tx;
@@ -2953,7 +2973,7 @@ pub(crate) mod test {
             details.received
         );
 
-        assert_eq!(details.fees, 200);
+        assert_eq!(details.fee.unwrap_or(0), 200);
     }
 
     #[test]
@@ -2962,7 +2982,7 @@ pub(crate) mod test {
         let addr = Address::from_str("2N1Ffz3WaNzbeLFBb51xyFMHYSEUXcbiSoX").unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .drain_wallet()
             .enable_rbf();
         let (psbt, mut original_details) = builder.finish().unwrap();
@@ -2986,18 +3006,18 @@ pub(crate) mod test {
         let mut builder = wallet.build_fee_bump(txid).unwrap();
         builder
             .fee_rate(FeeRate::from_sat_per_vb(2.5))
-            .maintain_single_recipient()
+            .allow_shrinking(addr.script_pubkey())
             .unwrap();
         let (psbt, details) = builder.finish().unwrap();
 
         assert_eq!(details.sent, original_details.sent);
-        assert!(details.fees > original_details.fees);
+        assert!(details.fee.unwrap_or(0) > original_details.fee.unwrap_or(0));
 
         let tx = &psbt.global.unsigned_tx;
         assert_eq!(tx.output.len(), 1);
-        assert_eq!(tx.output[0].value + details.fees, details.sent);
+        assert_eq!(tx.output[0].value + details.fee.unwrap_or(0), details.sent);
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::from_sat_per_vb(2.5), @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::from_sat_per_vb(2.5), @add_signature);
     }
 
     #[test]
@@ -3006,7 +3026,7 @@ pub(crate) mod test {
         let addr = Address::from_str("2N1Ffz3WaNzbeLFBb51xyFMHYSEUXcbiSoX").unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .drain_wallet()
             .enable_rbf();
         let (psbt, mut original_details) = builder.finish().unwrap();
@@ -3029,19 +3049,19 @@ pub(crate) mod test {
 
         let mut builder = wallet.build_fee_bump(txid).unwrap();
         builder
-            .maintain_single_recipient()
+            .allow_shrinking(addr.script_pubkey())
             .unwrap()
             .fee_absolute(300);
         let (psbt, details) = builder.finish().unwrap();
 
         assert_eq!(details.sent, original_details.sent);
-        assert!(details.fees > original_details.fees);
+        assert!(details.fee.unwrap_or(0) > original_details.fee.unwrap_or(0));
 
         let tx = &psbt.global.unsigned_tx;
         assert_eq!(tx.output.len(), 1);
-        assert_eq!(tx.output[0].value + details.fees, details.sent);
+        assert_eq!(tx.output[0].value + details.fee.unwrap_or(0), details.sent);
 
-        assert_eq!(details.fees, 300);
+        assert_eq!(details.fee.unwrap_or(0), 300);
     }
 
     #[test]
@@ -3060,7 +3080,7 @@ pub(crate) mod test {
         let addr = Address::from_str("2N1Ffz3WaNzbeLFBb51xyFMHYSEUXcbiSoX").unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .add_utxo(outpoint)
             .unwrap()
             .manually_selected_only()
@@ -3089,7 +3109,7 @@ pub(crate) mod test {
         let mut builder = wallet.build_fee_bump(txid).unwrap();
         builder
             .drain_wallet()
-            .maintain_single_recipient()
+            .allow_shrinking(addr.script_pubkey())
             .unwrap()
             .fee_rate(FeeRate::from_sat_per_vb(5.0));
         let (_, details) = builder.finish().unwrap();
@@ -3104,7 +3124,7 @@ pub(crate) mod test {
         // them, and make sure that `bump_fee` doesn't try to add more. This fails because we've
         // told the wallet it's not allowed to add more inputs AND it can't reduce the value of the
         // existing output. In other words, bump_fee + manually_selected_only is always an error
-        // unless you've also set "maintain_single_recipient".
+        // unless you've also set "allow_shrinking" OR there is a change output.
         let incoming_txid = crate::populate_test_db!(
             wallet.database.borrow_mut(),
             testutils! (@tx ( (@external descriptors, 0) => 25_000 ) (@confirmations 1)),
@@ -3117,7 +3137,7 @@ pub(crate) mod test {
         let addr = Address::from_str("2N1Ffz3WaNzbeLFBb51xyFMHYSEUXcbiSoX").unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .add_utxo(outpoint)
             .unwrap()
             .manually_selected_only()
@@ -3186,7 +3206,7 @@ pub(crate) mod test {
         let (psbt, details) = builder.finish().unwrap();
 
         assert_eq!(details.sent, original_details.sent + 25_000);
-        assert_eq!(details.fees + details.received, 30_000);
+        assert_eq!(details.fee.unwrap_or(0) + details.received, 30_000);
 
         let tx = &psbt.global.unsigned_tx;
         assert_eq!(tx.input.len(), 2);
@@ -3208,7 +3228,7 @@ pub(crate) mod test {
             details.received
         );
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::from_sat_per_vb(50.0), @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::from_sat_per_vb(50.0), @add_signature);
     }
 
     #[test]
@@ -3249,7 +3269,7 @@ pub(crate) mod test {
         let (psbt, details) = builder.finish().unwrap();
 
         assert_eq!(details.sent, original_details.sent + 25_000);
-        assert_eq!(details.fees + details.received, 30_000);
+        assert_eq!(details.fee.unwrap_or(0) + details.received, 30_000);
 
         let tx = &psbt.global.unsigned_tx;
         assert_eq!(tx.input.len(), 2);
@@ -3271,7 +3291,7 @@ pub(crate) mod test {
             details.received
         );
 
-        assert_eq!(details.fees, 6_000);
+        assert_eq!(details.fee.unwrap_or(0), 6_000);
     }
 
     #[test]
@@ -3283,11 +3303,11 @@ pub(crate) mod test {
             Some(100),
         );
 
-        // initially make a tx without change by using `set_single_recipient`
+        // initially make a tx without change by using `drain_to`
         let addr = Address::from_str("2N1Ffz3WaNzbeLFBb51xyFMHYSEUXcbiSoX").unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .add_utxo(OutPoint {
                 txid: incoming_txid,
                 vout: 0,
@@ -3315,17 +3335,17 @@ pub(crate) mod test {
             .set_tx(&original_details)
             .unwrap();
 
-        // now bump the fees without using `maintain_single_recipient`. the wallet should add an
+        // now bump the fees without using `allow_shrinking`. the wallet should add an
         // extra input and a change output, and leave the original output untouched
         let mut builder = wallet.build_fee_bump(txid).unwrap();
         builder.fee_rate(FeeRate::from_sat_per_vb(50.0));
         let (psbt, details) = builder.finish().unwrap();
 
-        let original_send_all_amount = original_details.sent - original_details.fees;
+        let original_send_all_amount = original_details.sent - original_details.fee.unwrap_or(0);
         assert_eq!(details.sent, original_details.sent + 50_000);
         assert_eq!(
             details.received,
-            75_000 - original_send_all_amount - details.fees
+            75_000 - original_send_all_amount - details.fee.unwrap_or(0)
         );
 
         let tx = &psbt.global.unsigned_tx;
@@ -3345,10 +3365,10 @@ pub(crate) mod test {
                 .find(|txout| txout.script_pubkey != addr.script_pubkey())
                 .unwrap()
                 .value,
-            75_000 - original_send_all_amount - details.fees
+            75_000 - original_send_all_amount - details.fee.unwrap_or(0)
         );
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::from_sat_per_vb(50.0), @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::from_sat_per_vb(50.0), @add_signature);
     }
 
     #[test]
@@ -3390,10 +3410,13 @@ pub(crate) mod test {
         builder.fee_rate(FeeRate::from_sat_per_vb(140.0));
         let (psbt, details) = builder.finish().unwrap();
 
-        assert_eq!(original_details.received, 5_000 - original_details.fees);
+        assert_eq!(
+            original_details.received,
+            5_000 - original_details.fee.unwrap_or(0)
+        );
 
         assert_eq!(details.sent, original_details.sent + 25_000);
-        assert_eq!(details.fees, 30_000);
+        assert_eq!(details.fee.unwrap_or(0), 30_000);
         assert_eq!(details.received, 0);
 
         let tx = &psbt.global.unsigned_tx;
@@ -3408,7 +3431,7 @@ pub(crate) mod test {
             45_000
         );
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::from_sat_per_vb(140.0), @dust_change, @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::from_sat_per_vb(140.0), @dust_change, @add_signature);
     }
 
     #[test]
@@ -3457,7 +3480,7 @@ pub(crate) mod test {
         let (psbt, details) = builder.finish().unwrap();
 
         assert_eq!(details.sent, original_details.sent + 25_000);
-        assert_eq!(details.fees + details.received, 30_000);
+        assert_eq!(details.fee.unwrap_or(0) + details.received, 30_000);
 
         let tx = &psbt.global.unsigned_tx;
         assert_eq!(tx.input.len(), 2);
@@ -3479,7 +3502,7 @@ pub(crate) mod test {
             details.received
         );
 
-        assert_fee_rate!(psbt.extract_tx(), details.fees, FeeRate::from_sat_per_vb(5.0), @add_signature);
+        assert_fee_rate!(psbt.extract_tx(), details.fee.unwrap_or(0), FeeRate::from_sat_per_vb(5.0), @add_signature);
     }
 
     #[test]
@@ -3528,7 +3551,7 @@ pub(crate) mod test {
         let (psbt, details) = builder.finish().unwrap();
 
         assert_eq!(details.sent, original_details.sent + 25_000);
-        assert_eq!(details.fees + details.received, 30_000);
+        assert_eq!(details.fee.unwrap_or(0) + details.received, 30_000);
 
         let tx = &psbt.global.unsigned_tx;
         assert_eq!(tx.input.len(), 2);
@@ -3550,7 +3573,7 @@ pub(crate) mod test {
             details.received
         );
 
-        assert_eq!(details.fees, 250);
+        assert_eq!(details.fee.unwrap_or(0), 250);
     }
 
     #[test]
@@ -3558,9 +3581,7 @@ pub(crate) mod test {
         let (wallet, _, _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (mut psbt, _) = builder.finish().unwrap();
 
         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
@@ -3575,9 +3596,7 @@ pub(crate) mod test {
         let (wallet, _, _) = get_funded_wallet("wpkh([d34db33f/84h/1h/0h]tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (mut psbt, _) = builder.finish().unwrap();
 
         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
@@ -3592,9 +3611,7 @@ pub(crate) mod test {
         let (wallet, _, _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/44'/0'/0'/0/*)");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (mut psbt, _) = builder.finish().unwrap();
 
         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
@@ -3609,9 +3626,7 @@ pub(crate) mod test {
         let (wallet, _, _) = get_funded_wallet("sh(wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*))");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (mut psbt, _) = builder.finish().unwrap();
 
         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
@@ -3627,9 +3642,7 @@ pub(crate) mod test {
             get_funded_wallet("wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (mut psbt, _) = builder.finish().unwrap();
 
         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
@@ -3644,9 +3657,7 @@ pub(crate) mod test {
         let (wallet, _, _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
-        builder
-            .set_single_recipient(addr.script_pubkey())
-            .drain_wallet();
+        builder.drain_to(addr.script_pubkey()).drain_wallet();
         let (mut psbt, _) = builder.finish().unwrap();
 
         psbt.inputs[0].bip32_derivation.clear();
@@ -3728,7 +3739,7 @@ pub(crate) mod test {
         let addr = wallet.get_address(New).unwrap();
         let mut builder = wallet.build_tx();
         builder
-            .set_single_recipient(addr.script_pubkey())
+            .drain_to(addr.script_pubkey())
             .sighash(sighash)
             .drain_wallet();
         let (mut psbt, _) = builder.finish().unwrap();

src/wallet/signer.rs

@@ -222,7 +222,7 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
             .bip32_derivation
             .iter()
             .filter_map(|(pk, &(fingerprint, ref path))| {
-                if self.matches(&(fingerprint, path.clone()), &secp).is_some() {
+                if self.matches(&(fingerprint, path.clone()), secp).is_some() {
                     Some((pk, path))
                 } else {
                     None
@@ -240,12 +240,12 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
                     &full_path.into_iter().cloned().collect::<Vec<ChildNumber>>()
                         [origin_path.len()..],
                 );
-                self.xkey.derive_priv(&secp, &deriv_path).unwrap()
+                self.xkey.derive_priv(secp, &deriv_path).unwrap()
             }
-            None => self.xkey.derive_priv(&secp, &full_path).unwrap(),
+            None => self.xkey.derive_priv(secp, &full_path).unwrap(),
         };
 
-        if &derived_key.private_key.public_key(&secp) != public_key {
+        if &derived_key.private_key.public_key(secp) != public_key {
             Err(SignerError::InvalidKey)
         } else {
             derived_key.private_key.sign(psbt, Some(input_index), secp)
@@ -257,7 +257,7 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
     }
 
     fn id(&self, secp: &SecpCtx) -> SignerId {
-        SignerId::from(self.root_fingerprint(&secp))
+        SignerId::from(self.root_fingerprint(secp))
     }
 
     fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
@@ -283,7 +283,7 @@ impl Signer for PrivateKey {
             return Ok(());
         }
 
-        let pubkey = self.public_key(&secp);
+        let pubkey = self.public_key(secp);
         if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
             return Ok(());
         }
@@ -591,7 +591,7 @@ impl ComputeSighash for Segwitv0 {
                     .map(Script::is_v0_p2wpkh)
                     .unwrap_or(false)
                 {
-                    p2wpkh_script_code(&psbt_input.redeem_script.as_ref().unwrap())
+                    p2wpkh_script_code(psbt_input.redeem_script.as_ref().unwrap())
                 } else {
                     return Err(SignerError::MissingWitnessScript);
                 }

src/wallet/tx_builder.rs

@@ -133,7 +133,7 @@ pub struct TxBuilder<'a, B, D, Cs, Ctx> {
 pub(crate) struct TxParams {
     pub(crate) recipients: Vec<(Script, u64)>,
     pub(crate) drain_wallet: bool,
-    pub(crate) single_recipient: Option<Script>,
+    pub(crate) drain_to: Option<Script>,
     pub(crate) fee_policy: Option<FeePolicy>,
     pub(crate) internal_policy_path: Option<BTreeMap<String, Vec<usize>>>,
     pub(crate) external_policy_path: Option<BTreeMap<String, Vec<usize>>>,
@@ -560,49 +560,81 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>> TxBuilder<'a, B, D,
         self
     }
 
-    /// Set a single recipient that will get all the selected funds minus the fee. No change will
-    /// be created
+    /// Sets the address to *drain* excess coins to.
     ///
-    /// This method overrides any recipient set with [`set_recipients`](Self::set_recipients) or
-    /// [`add_recipient`](Self::add_recipient).
+    /// Usually, when there are excess coins they are sent to a change address generated by the
+    /// wallet. This option replaces the usual change address with an arbitrary `script_pubkey` of
+    /// your choosing. Just as with a change output, if the drain output is not needed (the excess
+    /// coins are too small) it will not be included in the resulting transaction. The only
+    /// difference is that it is valid to use `drain_to` without setting any ordinary recipients
+    /// with [`add_recipient`] (but it is perfectly fine to add recipients as well).
     ///
-    /// It can only be used in conjunction with [`drain_wallet`](Self::drain_wallet) to send the
-    /// entire content of the wallet (minus filters) to a single recipient or with a
-    /// list of manually selected UTXOs by enabling [`manually_selected_only`](Self::manually_selected_only)
-    /// and selecting them with or [`add_utxo`](Self::add_utxo).
+    /// When bumping the fees of a transaction made with this option, you probably want to
+    /// use [`allow_shrinking`] to allow this output to be reduced to pay for the extra fees.
     ///
-    /// When bumping the fees of a transaction made with this option, the user should remeber to
-    /// add [`maintain_single_recipient`](Self::maintain_single_recipient) to correctly update the
-    /// single output instead of adding one more for the change.
-    pub fn set_single_recipient(&mut self, recipient: Script) -> &mut Self {
-        self.params.single_recipient = Some(recipient);
-        self.params.recipients.clear();
-
+    /// # Example
+    ///
+    /// `drain_to` is very useful for draining all the coins in a wallet with [`drain_wallet`] to a
+    /// single address.
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// # use bitcoin::*;
+    /// # use bdk::*;
+    /// # use bdk::wallet::tx_builder::CreateTx;
+    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// # let wallet = doctest_wallet!();
+    /// let mut tx_builder = wallet.build_tx();
+    ///
+    /// tx_builder
+    ///     // Spend all outputs in this wallet.
+    ///     .drain_wallet()
+    ///     // Send the excess (which is all the coins minus the fee) to this address.
+    ///     .drain_to(to_address.script_pubkey())
+    ///     .fee_rate(FeeRate::from_sat_per_vb(5.0))
+    ///     .enable_rbf();
+    /// let (psbt, tx_details) = tx_builder.finish()?;
+    /// # Ok::<(), bdk::Error>(())
+    /// ```
+    ///
+    /// [`allow_shrinking`]: Self::allow_shrinking
+    /// [`add_recipient`]: Self::add_recipient
+    /// [`drain_wallet`]: Self::drain_wallet
+    pub fn drain_to(&mut self, script_pubkey: Script) -> &mut Self {
+        self.params.drain_to = Some(script_pubkey);
         self
     }
 }
 
 // methods supported only by bump_fee
 impl<'a, B, D: BatchDatabase> TxBuilder<'a, B, D, DefaultCoinSelectionAlgorithm, BumpFee> {
-    /// Bump the fees of a transaction made with [`set_single_recipient`](Self::set_single_recipient)
+    /// Explicitly tells the wallet that it is allowed to reduce the fee of the output matching this
+    /// `script_pubkey` in order to bump the transaction fee. Without specifying this the wallet
+    /// will attempt to find a change output to shrink instead.
     ///
-    /// Unless extra inputs are specified with [`add_utxo`], this flag will make
-    /// `bump_fee` reduce the value of the existing output, or fail if it would be consumed
-    /// entirely given the higher new fee rate.
+    /// **Note** that the output may shrink to below the dust limit and therefore be removed. If it is
+    /// preserved then it is currently not guaranteed to be in the same position as it was
+    /// originally.
     ///
-    /// If extra inputs are added and they are not entirely consumed in fees, a change output will not
-    /// be added; the existing output will simply grow in value.
-    ///
-    /// Fails if the transaction has more than one outputs.
-    ///
-    /// [`add_utxo`]: Self::add_utxo
-    pub fn maintain_single_recipient(&mut self) -> Result<&mut Self, Error> {
-        let mut recipients = self.params.recipients.drain(..).collect::<Vec<_>>();
-        if recipients.len() != 1 {
-            return Err(Error::SingleRecipientMultipleOutputs);
+    /// Returns an `Err` if `script_pubkey` can't be found among the recipients of the
+    /// transaction we are bumping.
+    pub fn allow_shrinking(&mut self, script_pubkey: Script) -> Result<&mut Self, Error> {
+        match self
+            .params
+            .recipients
+            .iter()
+            .position(|(recipient_script, _)| *recipient_script == script_pubkey)
+        {
+            Some(position) => {
+                self.params.recipients.remove(position);
+                self.params.drain_to = Some(script_pubkey);
+                Ok(self)
+            }
+            None => Err(Error::Generic(format!(
+                "{} was not in the original transaction",
+                script_pubkey
+            ))),
         }
-        self.params.single_recipient = Some(recipients.pop().unwrap().0);
-        Ok(self)
     }
 }
 

src/wallet/verify.rs

@@ -0,0 +1,185 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Verify transactions against the consensus rules
+
+use std::collections::HashMap;
+use std::fmt;
+
+use bitcoin::consensus::serialize;
+use bitcoin::{OutPoint, Transaction, Txid};
+
+use crate::blockchain::Blockchain;
+use crate::database::Database;
+use crate::error::Error;
+
+/// Verify a transaction against the consensus rules
+///
+/// This function uses [`bitcoinconsensus`] to verify transactions by fetching the required data
+/// either from the [`Database`] or using the [`Blockchain`].
+///
+/// Depending on the [capabilities](crate::blockchain::Blockchain::get_capabilities) of the
+/// [`Blockchain`] backend, the method could fail when called with old "historical" transactions or
+/// with unconfirmed transactions that have been evicted from the backend's memory.
+pub fn verify_tx<D: Database, B: Blockchain>(
+    tx: &Transaction,
+    database: &D,
+    blockchain: &B,
+) -> Result<(), VerifyError> {
+    log::debug!("Verifying {}", tx.txid());
+
+    let serialized_tx = serialize(tx);
+    let mut tx_cache = HashMap::<_, Transaction>::new();
+
+    for (index, input) in tx.input.iter().enumerate() {
+        let prev_tx = if let Some(prev_tx) = tx_cache.get(&input.previous_output.txid) {
+            prev_tx.clone()
+        } else if let Some(prev_tx) = database.get_raw_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else if let Some(prev_tx) = blockchain.get_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else {
+            return Err(VerifyError::MissingInputTx(input.previous_output.txid));
+        };
+
+        let spent_output = prev_tx
+            .output
+            .get(input.previous_output.vout as usize)
+            .ok_or(VerifyError::InvalidInput(input.previous_output))?;
+
+        bitcoinconsensus::verify(
+            &spent_output.script_pubkey.to_bytes(),
+            spent_output.value,
+            &serialized_tx,
+            index,
+        )?;
+
+        // Since we have a local cache we might as well cache stuff from the db, as it will very
+        // likely decrease latency compared to reading from disk or performing an SQL query.
+        tx_cache.insert(prev_tx.txid(), prev_tx);
+    }
+
+    Ok(())
+}
+
+/// Error during validation of a tx agains the consensus rules
+#[derive(Debug)]
+pub enum VerifyError {
+    /// The transaction being spent is not available in the database or the blockchain client
+    MissingInputTx(Txid),
+    /// The transaction being spent doesn't have the requested output
+    InvalidInput(OutPoint),
+
+    /// Consensus error
+    Consensus(bitcoinconsensus::Error),
+
+    /// Generic error
+    ///
+    /// It has to be wrapped in a `Box` since `Error` has a variant that contains this enum
+    Global(Box<Error>),
+}
+
+impl fmt::Display for VerifyError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for VerifyError {}
+
+impl From<Error> for VerifyError {
+    fn from(other: Error) -> Self {
+        VerifyError::Global(Box::new(other))
+    }
+}
+impl_error!(bitcoinconsensus::Error, Consensus, VerifyError);
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashSet;
+
+    use bitcoin::consensus::encode::deserialize;
+    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::{Transaction, Txid};
+
+    use crate::blockchain::{Blockchain, Capability, Progress};
+    use crate::database::{BatchDatabase, BatchOperations, MemoryDatabase};
+    use crate::FeeRate;
+
+    use super::*;
+
+    struct DummyBlockchain;
+
+    impl Blockchain for DummyBlockchain {
+        fn get_capabilities(&self) -> HashSet<Capability> {
+            Default::default()
+        }
+        fn setup<D: BatchDatabase, P: 'static + Progress>(
+            &self,
+            _database: &mut D,
+            _progress_update: P,
+        ) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_tx(&self, _txid: &Txid) -> Result<Option<Transaction>, Error> {
+            Ok(None)
+        }
+        fn broadcast(&self, _tx: &Transaction) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_height(&self) -> Result<u32, Error> {
+            Ok(42)
+        }
+        fn estimate_fee(&self, _target: usize) -> Result<FeeRate, Error> {
+            Ok(FeeRate::default_min_relay_fee())
+        }
+    }
+
+    #[test]
+    fn test_verify_fail_unsigned_tx() {
+        // https://blockstream.info/tx/95da344585fcf2e5f7d6cbf2c3df2dcce84f9196f7a7bb901a43275cd6eb7c3f
+        let prev_tx: Transaction = deserialize(&Vec::<u8>::from_hex("020000000101192dea5e66d444380e106f8e53acb171703f00d43fb6b3ae88ca5644bdb7e1000000006b48304502210098328d026ce138411f957966c1cf7f7597ccbb170f5d5655ee3e9f47b18f6999022017c3526fc9147830e1340e04934476a3d1521af5b4de4e98baf49ec4c072079e01210276f847f77ec8dd66d78affd3c318a0ed26d89dab33fa143333c207402fcec352feffffff023d0ac203000000001976a9144bfbaf6afb76cc5771bc6404810d1cc041a6933988aca4b956050000000017a91494d5543c74a3ee98e0cf8e8caef5dc813a0f34b48768cb0700").unwrap()).unwrap();
+        // https://blockstream.info/tx/aca326a724eda9a461c10a876534ecd5ae7b27f10f26c3862fb996f80ea2d45d
+        let signed_tx: Transaction = deserialize(&Vec::<u8>::from_hex("02000000013f7cebd65c27431a90bba7f796914fe8cc2ddfc3f2cbd6f7e5f2fc854534da95000000006b483045022100de1ac3bcdfb0332207c4a91f3832bd2c2915840165f876ab47c5f8996b971c3602201c6c053d750fadde599e6f5c4e1963df0f01fc0d97815e8157e3d59fe09ca30d012103699b464d1d8bc9e47d4fb1cdaa89a1c5783d68363c4dbc4b524ed3d857148617feffffff02836d3c01000000001976a914fc25d6d5c94003bf5b0c7b640a248e2c637fcfb088ac7ada8202000000001976a914fbed3d9b11183209a57999d54d59f67c019e756c88ac6acb0700").unwrap()).unwrap();
+
+        let mut database = MemoryDatabase::new();
+        let blockchain = DummyBlockchain;
+
+        let mut unsigned_tx = signed_tx.clone();
+        for input in &mut unsigned_tx.input {
+            input.script_sig = Default::default();
+            input.witness = Default::default();
+        }
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail with missing input tx");
+        assert!(
+            matches!(result, Err(VerifyError::MissingInputTx(txid)) if txid == prev_tx.txid()),
+            "Error should be a `MissingInputTx` error"
+        );
+
+        // insert the prev_tx
+        database.set_raw_tx(&prev_tx).unwrap();
+
+        let result = verify_tx(&unsigned_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail since the TX is unsigned");
+        assert!(
+            matches!(result, Err(VerifyError::Consensus(_))),
+            "Error should be a `Consensus` error"
+        );
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(
+            result.is_ok(),
+            "Should work since the TX is correctly signed"
+        );
+    }
+}