Bump version to 0.13.1-dev

(cherry picked from commit d75d221540)

.github/workflows/code_coverage.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Update toolchain
         run: rustup update
       - name: Test
-        run: cargo test --features all-keys,compiler,esplora,compact_filters --no-default-features
+        run: cargo test --features all-keys,compiler,esplora,ureq,compact_filters --no-default-features
                     
       - id: coverage
         name: Generate coverage

.github/workflows/cont_integration.yml

@@ -10,18 +10,23 @@ jobs:
     strategy:
       matrix:
         rust:
-          - 1.51.0 # STABLE
+          - 1.56.0 # STABLE
           - 1.46.0 # MSRV
         features:
           - default
           - minimal
           - all-keys
-          - minimal,esplora
+          - minimal,use-esplora-ureq
           - key-value-db
           - electrum
           - compact_filters
-          - esplora,key-value-db,electrum
+          - esplora,ureq,key-value-db,electrum
           - compiler
+          - rpc
+          - verify
+          - async-interface
+          - use-esplora-reqwest
+          - sqlite
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -74,26 +79,20 @@ jobs:
         run: cargo test --features test-md-docs --no-default-features -- doctest::ReadmeDoctests
 
   test-blockchains:
-    name: Test ${{ matrix.blockchain.name }}
-    runs-on: ubuntu-16.04
+    name: Blockchain ${{ matrix.blockchain.features }}
+    runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
         blockchain:
           - name: electrum
-            container: bitcoindevkit/electrs
-            start: /root/electrs --network regtest --jsonrpc-import
+            features: test-electrum
+          - name: rpc
+            features: test-rpc
           - name: esplora
-            container: bitcoindevkit/esplora
-            start: /root/electrs --network regtest -vvv --cookie admin:passw --jsonrpc-import --electrum-rpc-addr=0.0.0.0:60401 --http-addr 0.0.0.0:3002
-    container: ${{ matrix.blockchain.container }}
-    env:
-      BDK_RPC_AUTH: USER_PASS
-      BDK_RPC_USER: admin
-      BDK_RPC_PASS: passw
-      BDK_RPC_URL: 127.0.0.1:18443
-      BDK_RPC_WALLET: bdk-test
-      BDK_ELECTRUM_URL: tcp://127.0.0.1:60401
-      BDK_ESPLORA_URL: http://127.0.0.1:3002
+            features: test-esplora,use-esplora-reqwest
+          - name: esplora
+            features: test-esplora,use-esplora-ureq    
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -105,26 +104,17 @@ jobs:
             ~/.cargo/git
             target
           key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
-      - name: get pkg-config # running eslpora tests seems to need this
-        run: apt update && apt install -y --fix-missing pkg-config libssl-dev
-      - name: Install rustup
-        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - name: Set default toolchain
-        run: $HOME/.cargo/bin/rustup default 1.51.0 # STABLE
-      - name: Set profile
-        run: $HOME/.cargo/bin/rustup set profile minimal
-      - name: Update toolchain
-        run: $HOME/.cargo/bin/rustup update
-      - name: Start core
-        run: ./ci/start-core.sh
-      - name: start ${{ matrix.blockchain.name }}
-        run: nohup ${{ matrix.blockchain.start }} & sleep 5
+      - name: Setup rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
       - name: Test
-        run: $HOME/.cargo/bin/cargo test --features ${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests
-
+        run: cargo test --no-default-features --features ${{ matrix.blockchain.features }} ${{ matrix.blockchain.name }}::bdk_blockchain_tests
+  
   check-wasm:
     name: Check WASM
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-20.04
     env:
       CC: clang-10
       CFLAGS: -I/usr/include
@@ -141,11 +131,11 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
         # Install a recent version of clang that supports wasm32
       - run: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
-      - run: sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" || exit 1
+      - run: sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" || exit 1
       - run: sudo apt-get update || exit 1
       - run: sudo apt-get install -y libclang-common-10-dev clang-10 libc6-dev-i386 || exit 1
       - name: Set default toolchain
-        run: rustup default 1.51.0 # STABLE
+        run: rustup default 1.56.0 # STABLE
       - name: Set profile
         run: rustup set profile minimal
       - name: Add target wasm32
@@ -153,7 +143,7 @@ jobs:
       - name: Update toolchain
         run: rustup update
       - name: Check
-        run: cargo check --target wasm32-unknown-unknown --features esplora --no-default-features
+        run: cargo check --target wasm32-unknown-unknown --features use-esplora-reqwest --no-default-features
 
   fmt:
     name: Rust fmt

.github/workflows/nightly_docs.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Update toolchain
         run: rustup update
       - name: Build docs
-        run: cargo rustdoc --verbose --features=compiler,electrum,esplora,compact_filters,key-value-db,all-keys -- --cfg docsrs -Dwarnings
+        run: cargo rustdoc --verbose --features=compiler,electrum,esplora,ureq,compact_filters,key-value-db,all-keys,sqlite -- --cfg docsrs -Dwarnings
       - name: Upload artifact
         uses: actions/upload-artifact@v2
         with:

CHANGELOG.md

@@ -6,6 +6,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [v0.13.0] - [v0.12.0]
+
+- Exposed `get_tx()` method from `Database` to `Wallet`.
+
+## [v0.12.0] - [v0.11.0]
+
+- Activate `miniscript/use-serde` feature to allow consumers of the library to access it via the re-exported `miniscript` crate.
+- Add support for proxies in `EsploraBlockchain`
+- Added `SqliteDatabase` that implements `Database` backed by a sqlite database using `rusqlite` crate.
+
+## [v0.11.0] - [v0.10.0]
+
+- Added `flush` method to the `Database` trait to explicitly flush to disk latest changes on the db.
+
+## [v0.10.0] - [v0.9.0]
+
+- Added `RpcBlockchain` in the `AnyBlockchain` struct to allow using Rpc backend where `AnyBlockchain` is used (eg `bdk-cli`)
+- Removed hard dependency on `tokio`.
+
+### Wallet
+
+- Removed and replaced `set_single_recipient` with more general `drain_to` and replaced `maintain_single_recipient` with `allow_shrinking`.
+
+### Blockchain
+
+- Removed `stop_gap` from `Blockchain` trait and added it to only `ElectrumBlockchain` and `EsploraBlockchain` structs.
+- Added a `ureq` backend for use when not using feature `async-interface` or target WASM. `ureq` is a blocking HTTP client.
+
+## [v0.9.0] - [v0.8.0]
+
+### Wallet
+
+- Added Bitcoin core RPC added as blockchain backend
+- Added a `verify` feature that can be enable to verify the unconfirmed txs we download against the consensus rules
+
 ## [v0.8.0] - [v0.7.0]
 
 ### Wallet
@@ -339,7 +374,7 @@ final transaction is created by calling `finish` on the builder.
 - Use `MemoryDatabase` in the compiler example
 - Make the REPL return JSON
 
-[unreleased]: https://github.com/bitcoindevkit/bdk/compare/v0.8.0...HEAD
+[unreleased]: https://github.com/bitcoindevkit/bdk/compare/v0.11.0...HEAD
 [0.1.0-beta.1]: https://github.com/bitcoindevkit/bdk/compare/96c87ea5...0.1.0-beta.1
 [v0.2.0]: https://github.com/bitcoindevkit/bdk/compare/0.1.0-beta.1...v0.2.0
 [v0.3.0]: https://github.com/bitcoindevkit/bdk/compare/v0.2.0...v0.3.0
@@ -348,4 +383,9 @@ final transaction is created by calling `finish` on the builder.
 [v0.5.1]: https://github.com/bitcoindevkit/bdk/compare/v0.5.0...v0.5.1
 [v0.6.0]: https://github.com/bitcoindevkit/bdk/compare/v0.5.1...v0.6.0
 [v0.7.0]: https://github.com/bitcoindevkit/bdk/compare/v0.6.0...v0.7.0
-[v0.8.0]: https://github.com/bitcoindevkit/bdk/compare/v0.7.0...v0.8.0
+[v0.8.0]: https://github.com/bitcoindevkit/bdk/compare/v0.7.0...v0.8.0
+[v0.9.0]: https://github.com/bitcoindevkit/bdk/compare/v0.8.0...v0.9.0
+[v0.10.0]: https://github.com/bitcoindevkit/bdk/compare/v0.9.0...v0.10.0
+[v0.11.0]: https://github.com/bitcoindevkit/bdk/compare/v0.10.0...v0.11.0
+[v0.12.0]: https://github.com/bitcoindevkit/bdk/compare/v0.11.0...v0.12.0
+[v0.13.0]: https://github.com/bitcoindevkit/bdk/compare/v0.12.0...v0.13.0

CONTRIBUTING.md

@@ -57,6 +57,21 @@ comment suggesting that you're working on it. If someone is already assigned,
 don't hesitate to ask if the assigned party or previous commenters are still
 working on it if it has been awhile.
 
+Deprecation policy
+------------------
+
+Where possible, breaking existing APIs should be avoided. Instead, add new APIs and
+use [`#[deprecated]`](https://github.com/rust-lang/rfcs/blob/master/text/1270-deprecation.md)
+to discourage use of the old one.
+
+Deprecated APIs are typically maintained for one release cycle. In other words, an
+API that has been deprecated with the 0.10 release can be expected to be removed in the
+0.11 release. This allows for smoother upgrades without incurring too much technical
+debt inside this library.
+
+If you deprecated an API as part of a contribution, we encourage you to "own" that API
+and send a follow-up to remove it as part of the next release cycle.
+
 Peer review
 -----------
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk"
-version = "0.8.1-dev"
+version = "0.13.1-dev"
 edition = "2018"
 authors = ["Alekos Filini <alekos.filini@gmail.com>", "Riccardo Casatta <riccardo@casatta.it>"]
 homepage = "https://bitcoindevkit.org"
@@ -12,29 +12,40 @@ readme = "README.md"
 license = "MIT OR Apache-2.0"
 
 [dependencies]
-bdk-macros = "^0.4"
+bdk-macros = "^0.6"
 log = "^0.4"
-miniscript = "5.1"
-bitcoin = { version = "^0.26", features = ["use-serde"] }
+miniscript = { version = "^6.0", features = ["use-serde"] }
+bitcoin = { version = "^0.27", features = ["use-serde", "base64"] }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
 rand = "^0.7"
 
 # Optional dependencies
 sled = { version = "0.34", optional = true }
-electrum-client = { version = "0.7", optional = true }
+electrum-client = { version = "0.8", optional = true }
+rusqlite = { version = "0.25.3", optional = true }
+ahash = { version = "=0.7.4", optional = true }
 reqwest = { version = "0.11", optional = true, features = ["json"] }
+ureq = { version = "~2.2.0", features = ["json"], optional = true }
 futures = { version = "0.3", optional = true }
 async-trait = { version = "0.1", optional = true }
-rocksdb = { version = "0.14", optional = true }
+rocksdb = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
 cc = { version = ">=1.0.64", optional = true }
 socks = { version = "0.3", optional = true }
 lazy_static = { version = "1.4", optional = true }
-tiny-bip39 = { version = "^0.8", optional = true }
+
+# the latest 0.8 version of tiny-bip39 depends on zeroize_derive 1.2 which has MSRV 1.51 and our 
+# MSRV is 1.46, to fix this until we update our MSRV or replace the tiny-bip39 
+# dependency https://github.com/bitcoindevkit/bdk/issues/399 we can only use an older version
+tiny-bip39 = { version = "< 0.8", optional = true }
+# backtrace > 0.3.61 includes object v0.27 which doesn't compile on 1.46. this is used by
+# tiny-bip39
+backtrace = { version = "=0.3.61", optional = true }
+
+bitcoinconsensus = { version = "0.19.0-3", optional = true }
 
 # Needed by bdk_blockchain_tests macro
-bitcoincore-rpc = {  version = "0.13", optional = true }
-serial_test = { version = "0.4", optional = true }
+bitcoincore-rpc = { version = "0.14", optional = true }
 
 # Platform-specific dependencies
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -48,25 +59,49 @@ rand = { version = "^0.7", features = ["wasm-bindgen"] }
 [features]
 minimal = []
 compiler = ["miniscript/compiler"]
+verify = ["bitcoinconsensus"]
 default = ["key-value-db", "electrum"]
-electrum = ["electrum-client"]
-esplora = ["reqwest", "futures"]
+sqlite = ["rusqlite", "ahash"]
 compact_filters = ["rocksdb", "socks", "lazy_static", "cc"]
 key-value-db = ["sled"]
-async-interface = ["async-trait"]
 all-keys = ["keys-bip39"]
-keys-bip39 = ["tiny-bip39"]
+keys-bip39 = ["tiny-bip39", "backtrace"]
+rpc = ["bitcoincore-rpc"]
+
+# We currently provide mulitple implementations of `Blockchain`, all are
+# blocking except for the `EsploraBlockchain` which can be either async or
+# blocking, depending on the HTTP client in use.
+#
+# - Users wanting asynchronous HTTP calls should enable `async-interface` to get
+#   access to the asynchronous method implementations. Then, if Esplora is wanted,
+#   enable `esplora` AND `reqwest` (`--features=use-esplora-reqwest`).
+# - Users wanting blocking HTTP calls can use any of the other blockchain
+#   implementations (`compact_filters`, `electrum`, or `esplora`). Users wanting to
+#   use Esplora should enable `esplora` AND `ureq`  (`--features=use-esplora-ureq`).
+#
+# WARNING: Please take care with the features below, various combinations will
+# fail to build. We cannot currently build `bdk` with `--all-features`.
+async-interface = ["async-trait"]
+electrum = ["electrum-client"]
+# MUST ALSO USE `--no-default-features`.
+use-esplora-reqwest = ["esplora", "reqwest", "reqwest/socks", "futures"]
+use-esplora-ureq = ["esplora", "ureq", "ureq/socks"]
+# Typical configurations will not need to use `esplora` feature directly.
+esplora = []
+
 
 # Debug/Test features
 test-blockchains = ["bitcoincore-rpc", "electrum-client"]
+test-electrum = ["electrum", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-rpc = ["rpc", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-esplora = ["electrsd/legacy", "electrsd/esplora_a33e97e1", "test-blockchains"]
 test-md-docs = ["electrum"]
 
 [dev-dependencies]
 lazy_static = "1.4"
 env_logger = "0.7"
-base64 = "^0.11"
 clap = "2.33"
-serial_test = "0.4"
+electrsd = { version= "0.12", features = ["trigger", "bitcoind_0_21_1"] }
 
 [[example]]
 name = "address_validator"
@@ -82,6 +117,6 @@ required-features = ["compiler"]
 [workspace]
 members = ["macros"]
 [package.metadata.docs.rs]
-features = ["compiler", "electrum", "esplora", "compact_filters", "key-value-db", "all-keys"]
+features = ["compiler", "electrum", "esplora", "ureq", "compact_filters", "rpc", "key-value-db", "sqlite", "all-keys", "verify"]
 # defines the configuration attribute `docsrs`
 rustdoc-args = ["--cfg", "docsrs"]

DEVELOPMENT_CYCLE.md

@@ -32,14 +32,14 @@ Pre-`v1.0.0` our "major" releases only affect the "minor" semver value. Accordin
     - If it's a minor issue you can just fix it in the release branch, since it will be merged back to `master` eventually
     - For bigger issues you can fix them on `master` and then *cherry-pick* the commit to the release branch
 6. Update the changelog with the new release version.
-7. Update `src/lib.rs` with the new version (line ~59)
+7. Update `src/lib.rs` with the new version (line ~43)
 8. On release day, make a commit on the release branch to bump the version to `x.y.z`. The message should be "Bump version to x.y.z".
 9. Add a tag to this commit. The tag name should be `vx.y.z` (for example `v0.5.0`), and the message "Release x.y.z". Make sure the tag is signed, for extra safety use the explicit `--sign` flag.
 10. Push the new commits to the upstream release branch, wait for the CI to finish one last time.
 11. Publish **all** the updated crates to crates.io.
 12. Make a new commit to bump the version value to `x.y.(z+1)-dev`. The message should be "Bump version to x.y.(z+1)-dev".
 13. Merge the release branch back into `master`.
-14. If the `master` branch contains any unreleased changes to the `bdk-macros`, `bdk-testutils`, or `bdk-testutils-macros` crates, change the `bdk` Cargo.toml `[dev-dependencies]` to point to the local path (ie. `bdk-testutils-macros = { path = "./testutils-macros"}`)
+14. If the `master` branch contains any unreleased changes to the `bdk-macros` crate, change the `bdk` Cargo.toml `[dependencies]` to point to the local path (ie. `bdk-macros = { path = "./macros"}`)
 15. Create the release on GitHub: go to "tags", click on the dots on the right and select "Create Release". Then set the title to `vx.y.z` and write down some brief release notes.
 16. Make sure the new release shows up on crates.io and that the docs are built correctly on docs.rs.
 17. Announce the release on Twitter, Discord and Telegram.

README.md

@@ -151,6 +151,25 @@ fn main() -> Result<(), bdk::Error> {
 }
 ```
 
+## Testing
+
+### Unit testing
+
+```
+cargo test
+```
+
+### Integration testing
+
+Integration testing require testing features, for example:
+
+```
+cargo test --features test-electrum
+```
+
+The other options are `test-esplora` or `test-rpc`.
+Note that `electrs` and `bitcoind` binaries are automatically downloaded (on mac and linux), to specify you already have installed binaries you must use `--no-default-features` and provide `BITCOIND_EXE` and `ELECTRS_EXE` as environment variables.
+
 ## License
 
 Licensed under either of

ci/start-core.sh

@@ -1,13 +1,14 @@
 #!/usr/bin/env sh
 
 echo "Starting bitcoin node."
-/root/bitcoind -regtest -server -daemon -fallbackfee=0.0002 -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -blockfilterindex=1 -peerblockfilters=1
+mkdir $GITHUB_WORKSPACE/.bitcoin
+/root/bitcoind -regtest -server -daemon -datadir=$GITHUB_WORKSPACE/.bitcoin -fallbackfee=0.0002 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -blockfilterindex=1 -peerblockfilters=1
 
 echo "Waiting for bitcoin node."
-until /root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS getblockchaininfo; do
+until /root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin getblockchaininfo; do
     sleep 1
 done
-/root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS createwallet $BDK_RPC_WALLET
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin createwallet $BDK_RPC_WALLET
 echo "Generating 150 bitcoin blocks."
-ADDR=$(/root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS -rpcwallet=$BDK_RPC_WALLET getnewaddress)
-/root/bitcoin-cli -regtest -rpcuser=$BDK_RPC_USER -rpcpassword=$BDK_RPC_PASS generatetoaddress 150 $ADDR
+ADDR=$(/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin -rpcwallet=$BDK_RPC_WALLET getnewaddress)
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin generatetoaddress 150 $ADDR

examples/compiler.rs

@@ -70,7 +70,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     let policy_str = matches.value_of("POLICY").unwrap();
     info!("Compiling policy: {}", policy_str);
 
-    let policy = Concrete::<String>::from_str(&policy_str)?;
+    let policy = Concrete::<String>::from_str(policy_str)?;
 
     let descriptor = match matches.value_of("TYPE").unwrap() {
         "sh" => Descriptor::new_sh(policy.compile()?)?,

macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk-macros"
-version = "0.4.0"
+version = "0.6.0"
 authors = ["Alekos Filini <alekos.filini@gmail.com>"]
 edition = "2018"
 homepage = "https://bitcoindevkit.org"

run_blockchain_tests.sh

@@ -1,64 +0,0 @@
-#!/bin/sh
-
-usage() {
-    cat <<'EOF'
-Script for running the bdk blockchain tests for a specific blockchain by starting up the backend in docker.
-
-Usage: ./run_blockchain_tests.sh [esplora|electrum] [test name].
-
-EOF
-}
-
-eprintln(){
-    echo "$@" >&2
-}
-
-cleanup() {
-    if test "$id"; then
-        eprintln "cleaning up $blockchain docker container $id";
-        docker rm -fv "$id" > /dev/null;
-    fi
-    trap - EXIT INT
-}
-
-# Makes sure we clean up the container at the end or if ^C
-trap 'rc=$?; cleanup; exit $rc' EXIT INT
-
-blockchain="$1"
-test_name="$2"
-
-case "$blockchain" in
-    electrum)
-        eprintln "starting electrs docker container"
-        id="$(docker run -d -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp bitcoindevkit/electrs)"
-        ;;
-    esplora)
-        eprintln "starting esplora docker container"
-        id="$(docker run -d -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp -p 127.0.0.1:3002:3002/tcp bitcoindevkit/esplora)"
-        export BDK_ESPLORA_URL=http://127.0.0.1:3002
-        ;;
-    *)
-        usage;
-        exit 1;
-        ;;
-    esac
-
-# taken from https://github.com/bitcoindevkit/bitcoin-regtest-box
-export BDK_RPC_AUTH=USER_PASS
-export BDK_RPC_USER=admin
-export BDK_RPC_PASS=passw
-export BDK_RPC_URL=127.0.0.1:18443
-export BDK_RPC_WALLET=bdk-test
-export BDK_ELECTRUM_URL=tcp://127.0.0.1:60401
-
-cli(){
-    docker exec -it "$id" /root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw $@
-}
-
-eprintln "running getwalletinfo until bitcoind seems to be alive"
-while ! cli getwalletinfo >/dev/null; do sleep 1; done
-
-# sleep again for good measure!
-sleep 1;
-
-cargo test --features "test-blockchains,$blockchain" --no-default-features "$blockchain::bdk_blockchain_tests::$test_name"

src/blockchain/any.rs

@@ -37,9 +37,9 @@
 //! )?;
 //! # }
 //!
-//! # #[cfg(feature = "esplora")]
+//! # #[cfg(all(feature = "esplora", feature = "ureq"))]
 //! # {
-//! let esplora_blockchain = EsploraBlockchain::new("...", None);
+//! let esplora_blockchain = EsploraBlockchain::new("...", 20);
 //! let wallet_esplora: Wallet<AnyBlockchain, _> = Wallet::new(
 //!     "...",
 //!     None,
@@ -60,6 +60,8 @@
 //! # use bdk::blockchain::*;
 //! # use bdk::database::MemoryDatabase;
 //! # use bdk::Wallet;
+//! # #[cfg(all(feature = "esplora", feature = "ureq"))]
+//! # {
 //! let config = serde_json::from_str("...")?;
 //! let blockchain = AnyBlockchain::from_config(&config)?;
 //! let wallet = Wallet::new(
@@ -69,6 +71,7 @@
 //!     MemoryDatabase::default(),
 //!     blockchain,
 //! )?;
+//! # }
 //! # Ok::<(), bdk::Error>(())
 //! ```
 
@@ -94,6 +97,8 @@ macro_rules! impl_inner_method {
             AnyBlockchain::Esplora(inner) => inner.$name( $($args, )* ),
             #[cfg(feature = "compact_filters")]
             AnyBlockchain::CompactFilters(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "rpc")]
+            AnyBlockchain::Rpc(inner) => inner.$name( $($args, )* ),
         }
     }
 }
@@ -116,6 +121,10 @@ pub enum AnyBlockchain {
     #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
     /// Compact filters client
     CompactFilters(compact_filters::CompactFiltersBlockchain),
+    #[cfg(feature = "rpc")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "rpc")))]
+    /// RPC client
+    Rpc(rpc::RpcBlockchain),
 }
 
 #[maybe_async]
@@ -126,31 +135,17 @@ impl Blockchain for AnyBlockchain {
 
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(impl_inner_method!(
-            self,
-            setup,
-            stop_gap,
-            database,
-            progress_update
-        ))
+        maybe_await!(impl_inner_method!(self, setup, database, progress_update))
     }
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(impl_inner_method!(
-            self,
-            sync,
-            stop_gap,
-            database,
-            progress_update
-        ))
+        maybe_await!(impl_inner_method!(self, sync, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
@@ -171,6 +166,7 @@ impl Blockchain for AnyBlockchain {
 impl_from!(electrum::ElectrumBlockchain, AnyBlockchain, Electrum, #[cfg(feature = "electrum")]);
 impl_from!(esplora::EsploraBlockchain, AnyBlockchain, Esplora, #[cfg(feature = "esplora")]);
 impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilters, #[cfg(feature = "compact_filters")]);
+impl_from!(rpc::RpcBlockchain, AnyBlockchain, Rpc, #[cfg(feature = "rpc")]);
 
 /// Type that can contain any of the blockchain configurations defined by the library
 ///
@@ -188,7 +184,8 @@ impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilt
 ///     r#"{
 ///    "type" : "electrum",
 ///    "url" : "ssl://electrum.blockstream.info:50002",
-///    "retry": 2
+///    "retry": 2,
+///    "stop_gap": 20
 /// }"#,
 /// )
 /// .unwrap();
@@ -198,7 +195,8 @@ impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilt
 ///         url: "ssl://electrum.blockstream.info:50002".into(),
 ///         retry: 2,
 ///         socks5: None,
-///         timeout: None
+///         timeout: None,
+///         stop_gap: 20,
 ///     })
 /// );
 /// # }
@@ -218,6 +216,10 @@ pub enum AnyBlockchainConfig {
     #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
     /// Compact filters client
     CompactFilters(compact_filters::CompactFiltersBlockchainConfig),
+    #[cfg(feature = "rpc")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "rpc")))]
+    /// RPC client configuration
+    Rpc(rpc::RpcConfig),
 }
 
 impl ConfigurableBlockchain for AnyBlockchain {
@@ -237,6 +239,10 @@ impl ConfigurableBlockchain for AnyBlockchain {
             AnyBlockchainConfig::CompactFilters(inner) => AnyBlockchain::CompactFilters(
                 compact_filters::CompactFiltersBlockchain::from_config(inner)?,
             ),
+            #[cfg(feature = "rpc")]
+            AnyBlockchainConfig::Rpc(inner) => {
+                AnyBlockchain::Rpc(rpc::RpcBlockchain::from_config(inner)?)
+            }
         })
     }
 }
@@ -244,3 +250,4 @@ impl ConfigurableBlockchain for AnyBlockchain {
 impl_from!(electrum::ElectrumBlockchainConfig, AnyBlockchainConfig, Electrum, #[cfg(feature = "electrum")]);
 impl_from!(esplora::EsploraBlockchainConfig, AnyBlockchainConfig, Esplora, #[cfg(feature = "esplora")]);
 impl_from!(compact_filters::CompactFiltersBlockchainConfig, AnyBlockchainConfig, CompactFilters, #[cfg(feature = "compact_filters")]);
+impl_from!(rpc::RpcConfig, AnyBlockchainConfig, Rpc, #[cfg(feature = "rpc")]);

src/blockchain/compact_filters/mod.rs

@@ -71,7 +71,7 @@ use super::{Blockchain, Capability, ConfigurableBlockchain, Progress};
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
 use crate::types::{KeychainKind, LocalUtxo, TransactionDetails};
-use crate::FeeRate;
+use crate::{ConfirmationTime, FeeRate};
 
 use peer::*;
 use store::*;
@@ -146,7 +146,7 @@ impl CompactFiltersBlockchain {
         database: &mut D,
         tx: &Transaction,
         height: Option<u32>,
-        timestamp: u64,
+        timestamp: Option<u64>,
         internal_max_deriv: &mut Option<u32>,
         external_max_deriv: &mut Option<u32>,
     ) -> Result<(), Error> {
@@ -206,9 +206,9 @@ impl CompactFiltersBlockchain {
                 transaction: Some(tx.clone()),
                 received: incoming,
                 sent: outgoing,
-                height,
-                timestamp,
-                fees: inputs_sum.saturating_sub(outputs_sum),
+                confirmation_time: ConfirmationTime::new(height, timestamp),
+                verified: height.is_some(),
+                fee: Some(inputs_sum.saturating_sub(outputs_sum)),
             };
 
             info!("Saving tx {}", tx.txid);
@@ -229,7 +229,6 @@ impl Blockchain for CompactFiltersBlockchain {
     #[allow(clippy::mutex_atomic)] // Mutex is easier to understand than a CAS loop.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        _stop_gap: Option<usize>, // TODO: move to electrum and esplora only
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
@@ -255,7 +254,7 @@ impl Blockchain for CompactFiltersBlockchain {
         let total_cost = headers_cost + filters_cost + PROCESS_BLOCKS_COST;
 
         if let Some(snapshot) = sync::sync_headers(
-            Arc::clone(&first_peer),
+            Arc::clone(first_peer),
             Arc::clone(&self.headers),
             |new_height| {
                 let local_headers_cost =
@@ -276,7 +275,7 @@ impl Blockchain for CompactFiltersBlockchain {
         let buried_height = synced_height.saturating_sub(sync::BURIED_CONFIRMATIONS);
         info!("Synced headers to height: {}", synced_height);
 
-        cf_sync.prepare_sync(Arc::clone(&first_peer))?;
+        cf_sync.prepare_sync(Arc::clone(first_peer))?;
 
         let all_scripts = Arc::new(
             database
@@ -295,7 +294,7 @@ impl Blockchain for CompactFiltersBlockchain {
         let mut threads = Vec::with_capacity(self.peers.len());
         for peer in &self.peers {
             let cf_sync = Arc::clone(&cf_sync);
-            let peer = Arc::clone(&peer);
+            let peer = Arc::clone(peer);
             let headers = Arc::clone(&self.headers);
             let all_scripts = Arc::clone(&all_scripts);
             let last_synced_block = Arc::clone(&last_synced_block);
@@ -364,8 +363,8 @@ impl Blockchain for CompactFiltersBlockchain {
         );
         let mut updates = database.begin_batch();
         for details in database.iter_txs(false)? {
-            match details.height {
-                Some(height) if (height as usize) < last_synced_block => continue,
+            match details.confirmation_time {
+                Some(c) if (c.height as usize) < last_synced_block => continue,
                 _ => updates.del_tx(&details.txid, false)?,
             };
         }
@@ -387,7 +386,7 @@ impl Blockchain for CompactFiltersBlockchain {
                     database,
                     tx,
                     Some(height as u32),
-                    0,
+                    None,
                     &mut internal_max_deriv,
                     &mut external_max_deriv,
                 )?;
@@ -398,7 +397,7 @@ impl Blockchain for CompactFiltersBlockchain {
                 database,
                 tx,
                 None,
-                0,
+                None,
                 &mut internal_max_deriv,
                 &mut external_max_deriv,
             )?;

src/blockchain/compact_filters/peer.rs

@@ -227,12 +227,12 @@ impl Peer {
 
         Ok(Peer {
             writer,
-            reader_thread,
             responses,
+            reader_thread,
             connected,
             mempool,
-            network,
             version,
+            network,
         })
     }
 
@@ -262,7 +262,7 @@ impl Peer {
         let message_resp = {
             let mut lock = responses.write().unwrap();
             let message_resp = lock.entry(wait_for).or_default();
-            Arc::clone(&message_resp)
+            Arc::clone(message_resp)
         };
 
         let (lock, cvar) = &*message_resp;
@@ -379,7 +379,7 @@ impl Peer {
             let message_resp = {
                 let mut lock = reader_thread_responses.write().unwrap();
                 let message_resp = lock.entry(in_message.cmd()).or_default();
-                Arc::clone(&message_resp)
+                Arc::clone(message_resp)
             };
 
             let (lock, cvar) = &*message_resp;

src/blockchain/compact_filters/store.rs

@@ -760,7 +760,7 @@ impl CfStore {
         let cf_headers: Vec<FilterHeader> = filter_hashes
             .into_iter()
             .scan(checkpoint, |prev_header, filter_hash| {
-                let filter_header = filter_hash.filter_header(&prev_header);
+                let filter_header = filter_hash.filter_header(prev_header);
                 *prev_header = filter_header;
 
                 Some(filter_header)
@@ -801,7 +801,7 @@ impl CfStore {
             .zip(headers.into_iter())
             .scan(checkpoint, |prev_header, ((_, filter_content), header)| {
                 let filter = BlockFilter::new(&filter_content);
-                if header != filter.filter_header(&prev_header) {
+                if header != filter.filter_header(prev_header) {
                     return Some(Err(CompactFiltersError::InvalidFilter));
                 }
                 *prev_header = header;

src/blockchain/compact_filters/sync.rs

@@ -205,7 +205,7 @@ impl CfSync {
                     let block_hash = self.headers_store.get_block_hash(height)?.unwrap();
 
                     // TODO: also download random blocks?
-                    if process(&block_hash, &BlockFilter::new(&filter))? {
+                    if process(&block_hash, &BlockFilter::new(filter))? {
                         log::debug!("Downloading block {}", block_hash);
 
                         let block = peer

src/blockchain/electrum.rs

@@ -43,11 +43,17 @@ use crate::FeeRate;
 ///
 /// ## Example
 /// See the [`blockchain::electrum`](crate::blockchain::electrum) module for a usage example.
-pub struct ElectrumBlockchain(Client);
+pub struct ElectrumBlockchain {
+    client: Client,
+    stop_gap: usize,
+}
 
 impl std::convert::From<Client> for ElectrumBlockchain {
     fn from(client: Client) -> Self {
-        ElectrumBlockchain(client)
+        ElectrumBlockchain {
+            client,
+            stop_gap: 20,
+        }
     }
 }
 
@@ -64,34 +70,33 @@ impl Blockchain for ElectrumBlockchain {
 
     fn setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        self.0
-            .electrum_like_setup(stop_gap, database, progress_update)
+        self.client
+            .electrum_like_setup(self.stop_gap, database, progress_update)
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self.0.transaction_get(txid).map(Option::Some)?)
+        Ok(self.client.transaction_get(txid).map(Option::Some)?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self.0.transaction_broadcast(tx).map(|_| ())?)
+        Ok(self.client.transaction_broadcast(tx).map(|_| ())?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
         // TODO: unsubscribe when added to the client, or is there a better call to use here?
 
         Ok(self
-            .0
+            .client
             .block_headers_subscribe()
             .map(|data| data.height as u32)?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
         Ok(FeeRate::from_btc_per_kvb(
-            self.0.estimate_fee(target)? as f32
+            self.client.estimate_fee(target)? as f32
         ))
     }
 }
@@ -149,6 +154,8 @@ pub struct ElectrumBlockchainConfig {
     pub retry: u8,
     /// Request timeout (seconds)
     pub timeout: Option<u8>,
+    /// Stop searching addresses for transactions after finding an unused gap of this length
+    pub stop_gap: usize,
 }
 
 impl ConfigurableBlockchain for ElectrumBlockchain {
@@ -162,16 +169,17 @@ impl ConfigurableBlockchain for ElectrumBlockchain {
             .socks5(socks5)?
             .build();
 
-        Ok(ElectrumBlockchain(Client::from_config(
-            config.url.as_str(),
-            electrum_config,
-        )?))
+        Ok(ElectrumBlockchain {
+            client: Client::from_config(config.url.as_str(), electrum_config)?,
+            stop_gap: config.stop_gap,
+        })
     }
 }
 
-#[cfg(feature = "test-blockchains")]
+#[cfg(test)]
+#[cfg(feature = "test-electrum")]
 crate::bdk_blockchain_tests! {
-    fn test_instance() -> ElectrumBlockchain {
-        ElectrumBlockchain::from(Client::new(&testutils::blockchain_tests::get_electrum_url()).unwrap())
+    fn test_instance(test_client: &TestClient) -> ElectrumBlockchain {
+        ElectrumBlockchain::from(Client::new(&test_client.electrsd.electrum_url).unwrap())
     }
 }

src/blockchain/esplora/mod.rs

@@ -0,0 +1,129 @@
+//! Esplora
+//!
+//! This module defines a [`EsploraBlockchain`] struct that can query an Esplora
+//! backend populate the wallet's [database](crate::database::Database) by:
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::esplora::EsploraBlockchain;
+//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", 20);
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! Esplora blockchain can use either `ureq` or `reqwest` for the HTTP client
+//! depending on your needs (blocking or async respectively).
+//!
+//! Please note, to configure the Esplora HTTP client correctly use one of:
+//! Blocking:  --features='esplora,ureq'
+//! Async:     --features='async-interface,esplora,reqwest' --no-default-features
+use std::collections::HashMap;
+use std::fmt;
+use std::io;
+
+use serde::Deserialize;
+
+use bitcoin::consensus;
+use bitcoin::{BlockHash, Txid};
+
+use crate::error::Error;
+use crate::FeeRate;
+
+#[cfg(feature = "reqwest")]
+mod reqwest;
+
+#[cfg(feature = "reqwest")]
+pub use self::reqwest::*;
+
+#[cfg(feature = "ureq")]
+mod ureq;
+
+#[cfg(feature = "ureq")]
+pub use self::ureq::*;
+
+fn into_fee_rate(target: usize, estimates: HashMap<String, f64>) -> Result<FeeRate, Error> {
+    let fee_val = estimates
+        .into_iter()
+        .map(|(k, v)| Ok::<_, std::num::ParseIntError>((k.parse::<usize>()?, v)))
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| Error::Generic(e.to_string()))?
+        .into_iter()
+        .take_while(|(k, _)| k <= &target)
+        .map(|(_, v)| v)
+        .last()
+        .unwrap_or(1.0);
+
+    Ok(FeeRate::from_sat_per_vb(fee_val as f32))
+}
+
+/// Data type used when fetching transaction history from Esplora.
+#[derive(Deserialize)]
+pub struct EsploraGetHistory {
+    txid: Txid,
+    status: EsploraGetHistoryStatus,
+}
+
+#[derive(Deserialize)]
+struct EsploraGetHistoryStatus {
+    block_height: Option<usize>,
+}
+
+/// Errors that can happen during a sync with [`EsploraBlockchain`]
+#[derive(Debug)]
+pub enum EsploraError {
+    /// Error during ureq HTTP request
+    #[cfg(feature = "ureq")]
+    Ureq(::ureq::Error),
+    /// Transport error during the ureq HTTP call
+    #[cfg(feature = "ureq")]
+    UreqTransport(::ureq::Transport),
+    /// Error during reqwest HTTP request
+    #[cfg(feature = "reqwest")]
+    Reqwest(::reqwest::Error),
+    /// HTTP response error
+    HttpResponse(u16),
+    /// IO error during ureq response read
+    Io(io::Error),
+    /// No header found in ureq response
+    NoHeader,
+    /// Invalid number returned
+    Parsing(std::num::ParseIntError),
+    /// Invalid Bitcoin data returned
+    BitcoinEncoding(bitcoin::consensus::encode::Error),
+    /// Invalid Hex data returned
+    Hex(bitcoin::hashes::hex::Error),
+
+    /// Transaction not found
+    TransactionNotFound(Txid),
+    /// Header height not found
+    HeaderHeightNotFound(u32),
+    /// Header hash not found
+    HeaderHashNotFound(BlockHash),
+}
+
+impl fmt::Display for EsploraError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for EsploraError {}
+
+#[cfg(feature = "ureq")]
+impl_error!(::ureq::Error, Ureq, EsploraError);
+#[cfg(feature = "ureq")]
+impl_error!(::ureq::Transport, UreqTransport, EsploraError);
+#[cfg(feature = "reqwest")]
+impl_error!(::reqwest::Error, Reqwest, EsploraError);
+impl_error!(io::Error, Io, EsploraError);
+impl_error!(std::num::ParseIntError, Parsing, EsploraError);
+impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
+impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
+
+#[cfg(test)]
+#[cfg(feature = "test-esplora")]
+crate::bdk_blockchain_tests! {
+    fn test_instance(test_client: &TestClient) -> EsploraBlockchain {
+        EsploraBlockchain::new(&format!("http://{}",test_client.electrsd.esplora_url.as_ref().unwrap()), 20)
+    }
+}

src/blockchain/esplora/reqwest.rs

@@ -9,38 +9,25 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-//! Esplora
-//!
-//! This module defines a [`Blockchain`] struct that can query an Esplora backend
-//! populate the wallet's [database](crate::database::Database) by
-//!
-//! ## Example
-//!
-//! ```no_run
-//! # use bdk::blockchain::esplora::EsploraBlockchain;
-//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", None);
-//! # Ok::<(), bdk::Error>(())
-//! ```
+//! Esplora by way of `reqwest` HTTP client.
 
 use std::collections::{HashMap, HashSet};
-use std::fmt;
 
-use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
+use bitcoin::consensus::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::{sha256, Hash};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use serde::Deserialize;
+use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
 
-use reqwest::{Client, StatusCode};
+use ::reqwest::{Client, StatusCode};
 
-use bitcoin::consensus::{self, deserialize, serialize};
-use bitcoin::hashes::hex::{FromHex, ToHex};
-use bitcoin::hashes::{sha256, Hash};
-use bitcoin::{BlockHash, BlockHeader, Script, Transaction, Txid};
-
-use self::utils::{ElectrumLikeSync, ElsGetHistoryRes};
-use super::*;
+use crate::blockchain::esplora::{EsploraError, EsploraGetHistory};
+use crate::blockchain::utils::{ElectrumLikeSync, ElsGetHistoryRes};
+use crate::blockchain::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::wallet::utils::ChunksIterator;
@@ -62,22 +49,37 @@ struct UrlClient {
 /// ## Example
 /// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
 #[derive(Debug)]
-pub struct EsploraBlockchain(UrlClient);
+pub struct EsploraBlockchain {
+    url_client: UrlClient,
+    stop_gap: usize,
+}
 
 impl std::convert::From<UrlClient> for EsploraBlockchain {
     fn from(url_client: UrlClient) -> Self {
-        EsploraBlockchain(url_client)
+        EsploraBlockchain {
+            url_client,
+            stop_gap: 20,
+        }
     }
 }
 
 impl EsploraBlockchain {
-    /// Create a new instance of the client from a base URL
-    pub fn new(base_url: &str, concurrency: Option<u8>) -> Self {
-        EsploraBlockchain(UrlClient {
-            url: base_url.to_string(),
-            client: Client::new(),
-            concurrency: concurrency.unwrap_or(DEFAULT_CONCURRENT_REQUESTS),
-        })
+    /// Create a new instance of the client from a base URL and `stop_gap`.
+    pub fn new(base_url: &str, stop_gap: usize) -> Self {
+        EsploraBlockchain {
+            url_client: UrlClient {
+                url: base_url.to_string(),
+                client: Client::new(),
+                concurrency: DEFAULT_CONCURRENT_REQUESTS,
+            },
+            stop_gap,
+        }
+    }
+
+    /// Set the concurrency to use when doing batch queries against the Esplora instance.
+    pub fn with_concurrency(mut self, concurrency: u8) -> Self {
+        self.url_client.concurrency = concurrency;
+        self
     }
 }
 
@@ -95,42 +97,29 @@ impl Blockchain for EsploraBlockchain {
 
     fn setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
         maybe_await!(self
-            .0
-            .electrum_like_setup(stop_gap, database, progress_update))
+            .url_client
+            .electrum_like_setup(self.stop_gap, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(await_or_block!(self.0._get_tx(txid))?)
+        Ok(await_or_block!(self.url_client._get_tx(txid))?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(await_or_block!(self.0._broadcast(tx))?)
+        Ok(await_or_block!(self.url_client._broadcast(tx))?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
-        Ok(await_or_block!(self.0._get_height())?)
+        Ok(await_or_block!(self.url_client._get_height())?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
-        let estimates = await_or_block!(self.0._get_fee_estimates())?;
-
-        let fee_val = estimates
-            .into_iter()
-            .map(|(k, v)| Ok::<_, std::num::ParseIntError>((k.parse::<usize>()?, v)))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|e| Error::Generic(e.to_string()))?
-            .into_iter()
-            .take_while(|(k, _)| k <= &target)
-            .map(|(_, v)| v)
-            .last()
-            .unwrap_or(1.0);
-
-        Ok(FeeRate::from_sat_per_vb(fee_val as f32))
+        let estimates = await_or_block!(self.url_client._get_fee_estimates())?;
+        super::into_fee_rate(target, estimates)
     }
 }
 
@@ -292,74 +281,51 @@ impl ElectrumLikeSync for UrlClient {
         &self,
         scripts: I,
     ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error> {
-        let future = async {
-            let mut results = vec![];
-            for chunk in ChunksIterator::new(scripts.into_iter(), self.concurrency as usize) {
-                let mut futs = FuturesOrdered::new();
-                for script in chunk {
-                    futs.push(self._script_get_history(&script));
-                }
-                let partial_results: Vec<Vec<ElsGetHistoryRes>> = futs.try_collect().await?;
-                results.extend(partial_results);
+        let mut results = vec![];
+        for chunk in ChunksIterator::new(scripts.into_iter(), self.concurrency as usize) {
+            let mut futs = FuturesOrdered::new();
+            for script in chunk {
+                futs.push(self._script_get_history(script));
             }
-            Ok(stream::iter(results).collect().await)
-        };
-
-        await_or_block!(future)
+            let partial_results: Vec<Vec<ElsGetHistoryRes>> = await_or_block!(futs.try_collect())?;
+            results.extend(partial_results);
+        }
+        Ok(await_or_block!(stream::iter(results).collect()))
     }
 
     fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
         &self,
         txids: I,
     ) -> Result<Vec<Transaction>, Error> {
-        let future = async {
-            let mut results = vec![];
-            for chunk in ChunksIterator::new(txids.into_iter(), self.concurrency as usize) {
-                let mut futs = FuturesOrdered::new();
-                for txid in chunk {
-                    futs.push(self._get_tx_no_opt(&txid));
-                }
-                let partial_results: Vec<Transaction> = futs.try_collect().await?;
-                results.extend(partial_results);
+        let mut results = vec![];
+        for chunk in ChunksIterator::new(txids.into_iter(), self.concurrency as usize) {
+            let mut futs = FuturesOrdered::new();
+            for txid in chunk {
+                futs.push(self._get_tx_no_opt(txid));
             }
-            Ok(stream::iter(results).collect().await)
-        };
-
-        await_or_block!(future)
+            let partial_results: Vec<Transaction> = await_or_block!(futs.try_collect())?;
+            results.extend(partial_results);
+        }
+        Ok(await_or_block!(stream::iter(results).collect()))
     }
 
     fn els_batch_block_header<I: IntoIterator<Item = u32>>(
         &self,
         heights: I,
     ) -> Result<Vec<BlockHeader>, Error> {
-        let future = async {
-            let mut results = vec![];
-            for chunk in ChunksIterator::new(heights.into_iter(), self.concurrency as usize) {
-                let mut futs = FuturesOrdered::new();
-                for height in chunk {
-                    futs.push(self._get_header(height));
-                }
-                let partial_results: Vec<BlockHeader> = futs.try_collect().await?;
-                results.extend(partial_results);
+        let mut results = vec![];
+        for chunk in ChunksIterator::new(heights.into_iter(), self.concurrency as usize) {
+            let mut futs = FuturesOrdered::new();
+            for height in chunk {
+                futs.push(self._get_header(height));
             }
-            Ok(stream::iter(results).collect().await)
-        };
-
-        await_or_block!(future)
+            let partial_results: Vec<BlockHeader> = await_or_block!(futs.try_collect())?;
+            results.extend(partial_results);
+        }
+        Ok(await_or_block!(stream::iter(results).collect()))
     }
 }
 
-#[derive(Deserialize)]
-struct EsploraGetHistoryStatus {
-    block_height: Option<usize>,
-}
-
-#[derive(Deserialize)]
-struct EsploraGetHistory {
-    txid: Txid,
-    status: EsploraGetHistoryStatus,
-}
-
 /// Configuration for an [`EsploraBlockchain`]
 #[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
 pub struct EsploraBlockchainConfig {
@@ -367,57 +333,40 @@ pub struct EsploraBlockchainConfig {
     ///
     /// eg. `https://blockstream.info/api/`
     pub base_url: String,
+    /// Optional URL of the proxy to use to make requests to the Esplora server
+    ///
+    /// The string should be formatted as: `<protocol>://<user>:<password>@host:<port>`.
+    ///
+    /// Note that the format of this value and the supported protocols change slightly between the
+    /// sync version of esplora (using `ureq`) and the async version (using `reqwest`). For more
+    /// details check with the documentation of the two crates. Both of them are compiled with
+    /// the `socks` feature enabled.
+    ///
+    /// The proxy is ignored when targeting `wasm32`.
+    pub proxy: Option<String>,
     /// Number of parallel requests sent to the esplora service (default: 4)
     pub concurrency: Option<u8>,
+    /// Stop searching addresses for transactions after finding an unused gap of this length.
+    pub stop_gap: usize,
 }
 
 impl ConfigurableBlockchain for EsploraBlockchain {
     type Config = EsploraBlockchainConfig;
 
     fn from_config(config: &Self::Config) -> Result<Self, Error> {
-        Ok(EsploraBlockchain::new(
-            config.base_url.as_str(),
-            config.concurrency,
-        ))
-    }
-}
-
-/// Errors that can happen during a sync with [`EsploraBlockchain`]
-#[derive(Debug)]
-pub enum EsploraError {
-    /// Error with the HTTP call
-    Reqwest(reqwest::Error),
-    /// Invalid number returned
-    Parsing(std::num::ParseIntError),
-    /// Invalid Bitcoin data returned
-    BitcoinEncoding(bitcoin::consensus::encode::Error),
-    /// Invalid Hex data returned
-    Hex(bitcoin::hashes::hex::Error),
-
-    /// Transaction not found
-    TransactionNotFound(Txid),
-    /// Header height not found
-    HeaderHeightNotFound(u32),
-    /// Header hash not found
-    HeaderHashNotFound(BlockHash),
-}
-
-impl fmt::Display for EsploraError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
-
-impl std::error::Error for EsploraError {}
-
-impl_error!(reqwest::Error, Reqwest, EsploraError);
-impl_error!(std::num::ParseIntError, Parsing, EsploraError);
-impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
-impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
-
-#[cfg(feature = "test-blockchains")]
-crate::bdk_blockchain_tests! {
-    fn test_instance() -> EsploraBlockchain {
-        EsploraBlockchain::new(std::env::var("BDK_ESPLORA_URL").unwrap_or("127.0.0.1:3002".into()).as_str(), None)
+        let map_e = |e: reqwest::Error| Error::Esplora(Box::new(e.into()));
+
+        let mut blockchain = EsploraBlockchain::new(config.base_url.as_str(), config.stop_gap);
+        if let Some(concurrency) = config.concurrency {
+            blockchain.url_client.concurrency = concurrency;
+        }
+        #[cfg(not(target_arch = "wasm32"))]
+        if let Some(proxy) = &config.proxy {
+            blockchain.url_client.client = Client::builder()
+                .proxy(reqwest::Proxy::all(proxy).map_err(map_e)?)
+                .build()
+                .map_err(map_e)?;
+        }
+        Ok(blockchain)
     }
 }

src/blockchain/esplora/ureq.rs

@@ -0,0 +1,398 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Esplora by way of `ureq` HTTP client.
+
+use std::collections::{HashMap, HashSet};
+use std::io;
+use std::io::Read;
+use std::time::Duration;
+
+#[allow(unused_imports)]
+use log::{debug, error, info, trace};
+
+use ureq::{Agent, Proxy, Response};
+
+use bitcoin::consensus::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::{sha256, Hash};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
+
+use crate::blockchain::esplora::{EsploraError, EsploraGetHistory};
+use crate::blockchain::utils::{ElectrumLikeSync, ElsGetHistoryRes};
+use crate::blockchain::*;
+use crate::database::BatchDatabase;
+use crate::error::Error;
+use crate::FeeRate;
+
+#[derive(Debug)]
+struct UrlClient {
+    url: String,
+    agent: Agent,
+}
+
+/// Structure that implements the logic to sync with Esplora
+///
+/// ## Example
+/// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
+#[derive(Debug)]
+pub struct EsploraBlockchain {
+    url_client: UrlClient,
+    stop_gap: usize,
+}
+
+impl std::convert::From<UrlClient> for EsploraBlockchain {
+    fn from(url_client: UrlClient) -> Self {
+        EsploraBlockchain {
+            url_client,
+            stop_gap: 20,
+        }
+    }
+}
+
+impl EsploraBlockchain {
+    /// Create a new instance of the client from a base URL and the `stop_gap`.
+    pub fn new(base_url: &str, stop_gap: usize) -> Self {
+        EsploraBlockchain {
+            url_client: UrlClient {
+                url: base_url.to_string(),
+                agent: Agent::new(),
+            },
+            stop_gap,
+        }
+    }
+
+    /// Set the inner `ureq` agent.
+    pub fn with_agent(mut self, agent: Agent) -> Self {
+        self.url_client.agent = agent;
+        self
+    }
+}
+
+impl Blockchain for EsploraBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        vec![
+            Capability::FullHistory,
+            Capability::GetAnyTx,
+            Capability::AccurateFees,
+        ]
+        .into_iter()
+        .collect()
+    }
+
+    fn setup<D: BatchDatabase, P: Progress>(
+        &self,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        self.url_client
+            .electrum_like_setup(self.stop_gap, database, progress_update)
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(self.url_client._get_tx(txid)?)
+    }
+
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        let _txid = self.url_client._broadcast(tx)?;
+        Ok(())
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(self.url_client._get_height()?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        let estimates = self.url_client._get_fee_estimates()?;
+        super::into_fee_rate(target, estimates)
+    }
+}
+
+impl UrlClient {
+    fn script_to_scripthash(script: &Script) -> String {
+        sha256::Hash::hash(script.as_bytes()).into_inner().to_hex()
+    }
+
+    fn _get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/tx/{}/raw", self.url, txid))
+            .call();
+
+        match resp {
+            Ok(resp) => Ok(Some(deserialize(&into_bytes(resp)?)?)),
+            Err(ureq::Error::Status(code, _)) => {
+                if is_status_not_found(code) {
+                    return Ok(None);
+                }
+                Err(EsploraError::HttpResponse(code))
+            }
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _get_tx_no_opt(&self, txid: &Txid) -> Result<Transaction, EsploraError> {
+        match self._get_tx(txid) {
+            Ok(Some(tx)) => Ok(tx),
+            Ok(None) => Err(EsploraError::TransactionNotFound(*txid)),
+            Err(e) => Err(e),
+        }
+    }
+
+    fn _get_header(&self, block_height: u32) -> Result<BlockHeader, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/block-height/{}", self.url, block_height))
+            .call();
+
+        let bytes = match resp {
+            Ok(resp) => Ok(into_bytes(resp)?),
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }?;
+
+        let hash = std::str::from_utf8(&bytes)
+            .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?;
+
+        let resp = self
+            .agent
+            .get(&format!("{}/block/{}/header", self.url, hash))
+            .call();
+
+        match resp {
+            Ok(resp) => Ok(deserialize(&Vec::from_hex(&resp.into_string()?)?)?),
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
+        let resp = self
+            .agent
+            .post(&format!("{}/tx", self.url))
+            .send_string(&serialize(transaction).to_hex());
+
+        match resp {
+            Ok(_) => Ok(()), // We do not return the txid?
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _get_height(&self) -> Result<u32, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/blocks/tip/height", self.url))
+            .call();
+
+        match resp {
+            Ok(resp) => Ok(resp.into_string()?.parse()?),
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _script_get_history(&self, script: &Script) -> Result<Vec<ElsGetHistoryRes>, EsploraError> {
+        let mut result = Vec::new();
+        let scripthash = Self::script_to_scripthash(script);
+
+        // Add the unconfirmed transactions first
+
+        let resp = self
+            .agent
+            .get(&format!(
+                "{}/scripthash/{}/txs/mempool",
+                self.url, scripthash
+            ))
+            .call();
+
+        let v = match resp {
+            Ok(resp) => {
+                let v: Vec<EsploraGetHistory> = resp.into_json()?;
+                Ok(v)
+            }
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }?;
+
+        result.extend(v.into_iter().map(|x| ElsGetHistoryRes {
+            tx_hash: x.txid,
+            height: x.status.block_height.unwrap_or(0) as i32,
+        }));
+
+        debug!(
+            "Found {} mempool txs for {} - {:?}",
+            result.len(),
+            scripthash,
+            script
+        );
+
+        // Then go through all the pages of confirmed transactions
+        let mut last_txid = String::new();
+        loop {
+            let resp = self
+                .agent
+                .get(&format!(
+                    "{}/scripthash/{}/txs/chain/{}",
+                    self.url, scripthash, last_txid
+                ))
+                .call();
+
+            let v = match resp {
+                Ok(resp) => {
+                    let v: Vec<EsploraGetHistory> = resp.into_json()?;
+                    Ok(v)
+                }
+                Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+                Err(e) => Err(EsploraError::Ureq(e)),
+            }?;
+
+            let len = v.len();
+            if let Some(elem) = v.last() {
+                last_txid = elem.txid.to_hex();
+            }
+
+            debug!("... adding {} confirmed transactions", len);
+
+            result.extend(v.into_iter().map(|x| ElsGetHistoryRes {
+                tx_hash: x.txid,
+                height: x.status.block_height.unwrap_or(0) as i32,
+            }));
+
+            if len < 25 {
+                break;
+            }
+        }
+
+        Ok(result)
+    }
+
+    fn _get_fee_estimates(&self) -> Result<HashMap<String, f64>, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/fee-estimates", self.url,))
+            .call();
+
+        let map = match resp {
+            Ok(resp) => {
+                let map: HashMap<String, f64> = resp.into_json()?;
+                Ok(map)
+            }
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }?;
+
+        Ok(map)
+    }
+}
+
+fn is_status_not_found(status: u16) -> bool {
+    status == 404
+}
+
+fn into_bytes(resp: Response) -> Result<Vec<u8>, io::Error> {
+    const BYTES_LIMIT: usize = 10 * 1_024 * 1_024;
+
+    let mut buf: Vec<u8> = vec![];
+    resp.into_reader()
+        .take((BYTES_LIMIT + 1) as u64)
+        .read_to_end(&mut buf)?;
+    if buf.len() > BYTES_LIMIT {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            "response too big for into_bytes",
+        ));
+    }
+
+    Ok(buf)
+}
+
+impl ElectrumLikeSync for UrlClient {
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
+        &self,
+        scripts: I,
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error> {
+        let mut results = vec![];
+        for script in scripts.into_iter() {
+            let v = self._script_get_history(script)?;
+            results.push(v);
+        }
+        Ok(results)
+    }
+
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
+        &self,
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut results = vec![];
+        for txid in txids.into_iter() {
+            let tx = self._get_tx_no_opt(txid)?;
+            results.push(tx);
+        }
+        Ok(results)
+    }
+
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        let mut results = vec![];
+        for height in heights.into_iter() {
+            let header = self._get_header(height)?;
+            results.push(header);
+        }
+        Ok(results)
+    }
+}
+
+/// Configuration for an [`EsploraBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
+pub struct EsploraBlockchainConfig {
+    /// Base URL of the esplora service eg. `https://blockstream.info/api/`
+    pub base_url: String,
+    /// Optional URL of the proxy to use to make requests to the Esplora server
+    ///
+    /// The string should be formatted as: `<protocol>://<user>:<password>@host:<port>`.
+    ///
+    /// Note that the format of this value and the supported protocols change slightly between the
+    /// sync version of esplora (using `ureq`) and the async version (using `reqwest`). For more
+    /// details check with the documentation of the two crates. Both of them are compiled with
+    /// the `socks` feature enabled.
+    ///
+    /// The proxy is ignored when targeting `wasm32`.
+    pub proxy: Option<String>,
+    /// Socket read timeout.
+    pub timeout_read: u64,
+    /// Socket write timeout.
+    pub timeout_write: u64,
+    /// Stop searching addresses for transactions after finding an unused gap of this length.
+    pub stop_gap: usize,
+}
+
+impl ConfigurableBlockchain for EsploraBlockchain {
+    type Config = EsploraBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let mut agent_builder = ureq::AgentBuilder::new()
+            .timeout_read(Duration::from_secs(config.timeout_read))
+            .timeout_write(Duration::from_secs(config.timeout_write));
+
+        if let Some(proxy) = &config.proxy {
+            agent_builder = agent_builder
+                .proxy(Proxy::new(proxy).map_err(|e| Error::Esplora(Box::new(e.into())))?);
+        }
+
+        Ok(
+            EsploraBlockchain::new(config.base_url.as_str(), config.stop_gap)
+                .with_agent(agent_builder.build()),
+        )
+    }
+}

src/blockchain/mod.rs

@@ -30,9 +30,19 @@ use crate::FeeRate;
 #[cfg(any(feature = "electrum", feature = "esplora"))]
 pub(crate) mod utils;
 
-#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+#[cfg(any(
+    feature = "electrum",
+    feature = "esplora",
+    feature = "compact_filters",
+    feature = "rpc"
+))]
 pub mod any;
-#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+#[cfg(any(
+    feature = "electrum",
+    feature = "esplora",
+    feature = "compact_filters",
+    feature = "rpc"
+))]
 pub use any::{AnyBlockchain, AnyBlockchainConfig};
 
 #[cfg(feature = "electrum")]
@@ -43,6 +53,14 @@ pub use self::electrum::ElectrumBlockchain;
 #[cfg(feature = "electrum")]
 pub use self::electrum::ElectrumBlockchainConfig;
 
+#[cfg(feature = "rpc")]
+#[cfg_attr(docsrs, doc(cfg(feature = "rpc")))]
+pub mod rpc;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcBlockchain;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcConfig;
+
 #[cfg(feature = "esplora")]
 #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
 pub mod esplora;
@@ -52,6 +70,7 @@ pub use self::esplora::EsploraBlockchain;
 #[cfg(feature = "compact_filters")]
 #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
 pub mod compact_filters;
+
 #[cfg(feature = "compact_filters")]
 pub use self::compact_filters::CompactFiltersBlockchain;
 
@@ -84,7 +103,6 @@ pub trait Blockchain {
     /// [`Blockchain::sync`] defaults to calling this internally if not overridden.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error>;
@@ -109,11 +127,10 @@ pub trait Blockchain {
     /// [`BatchOperations::del_utxo`]: crate::database::BatchOperations::del_utxo
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.setup(stop_gap, database, progress_update))
+        maybe_await!(self.setup(database, progress_update))
     }
 
     /// Fetch a transaction from the blockchain given its txid
@@ -166,7 +183,7 @@ impl Progress for Sender<ProgressData> {
 }
 
 /// Type that implements [`Progress`] and drops every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct NoopProgress;
 
 /// Create a new instance of [`NoopProgress`]
@@ -181,10 +198,10 @@ impl Progress for NoopProgress {
 }
 
 /// Type that implements [`Progress`] and logs at level `INFO` every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct LogProgress;
 
-/// Create a nwe instance of [`LogProgress`]
+/// Create a new instance of [`LogProgress`]
 pub fn log_progress() -> LogProgress {
     LogProgress
 }
@@ -209,20 +226,18 @@ impl<T: Blockchain> Blockchain for Arc<T> {
 
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.deref().setup(stop_gap, database, progress_update))
+        maybe_await!(self.deref().setup(database, progress_update))
     }
 
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.deref().sync(stop_gap, database, progress_update))
+        maybe_await!(self.deref().sync(database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {

src/blockchain/rpc.rs

@@ -0,0 +1,477 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Riccardo Casatta <riccardo@casatta.it>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Rpc Blockchain
+//!
+//! Backend that gets blockchain data from Bitcoin Core RPC
+//!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::{RpcConfig, RpcBlockchain, ConfigurableBlockchain, rpc::Auth};
+//! let config = RpcConfig {
+//!     url: "127.0.0.1:18332".to_string(),
+//!     auth: Auth::Cookie {
+//!         file: "/home/user/.bitcoin/.cookie".into(),
+//!     },
+//!     network: bdk::bitcoin::Network::Testnet,
+//!     wallet_name: "wallet_name".to_string(),
+//!     skip_blocks: None,
+//! };
+//! let blockchain = RpcBlockchain::from_config(&config);
+//! ```
+
+use crate::bitcoin::consensus::deserialize;
+use crate::bitcoin::{Address, Network, OutPoint, Transaction, TxOut, Txid};
+use crate::blockchain::{Blockchain, Capability, ConfigurableBlockchain, Progress};
+use crate::database::{BatchDatabase, DatabaseUtils};
+use crate::descriptor::{get_checksum, IntoWalletDescriptor};
+use crate::wallet::utils::SecpCtx;
+use crate::{ConfirmationTime, Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails};
+use bitcoincore_rpc::json::{
+    GetAddressInfoResultLabel, ImportMultiOptions, ImportMultiRequest,
+    ImportMultiRequestScriptPubkey, ImportMultiRescanSince,
+};
+use bitcoincore_rpc::jsonrpc::serde_json::Value;
+use bitcoincore_rpc::Auth as RpcAuth;
+use bitcoincore_rpc::{Client, RpcApi};
+use log::debug;
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+use std::str::FromStr;
+
+/// The main struct for RPC backend implementing the [crate::blockchain::Blockchain] trait
+#[derive(Debug)]
+pub struct RpcBlockchain {
+    /// Rpc client to the node, includes the wallet name
+    client: Client,
+    /// Network used
+    network: Network,
+    /// Blockchain capabilities, cached here at startup
+    capabilities: HashSet<Capability>,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    skip_blocks: Option<u32>,
+
+    /// This is a fixed Address used as a hack key to store information on the node
+    _storage_address: Address,
+}
+
+/// RpcBlockchain configuration options
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+pub struct RpcConfig {
+    /// The bitcoin node url
+    pub url: String,
+    /// The bitcoin node authentication mechanism
+    pub auth: Auth,
+    /// The network we are using (it will be checked the bitcoin node network matches this)
+    pub network: Network,
+    /// The wallet name in the bitcoin node, consider using [wallet_name_from_descriptor] for this
+    pub wallet_name: String,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    pub skip_blocks: Option<u32>,
+}
+
+/// This struct is equivalent to [bitcoincore_rpc::Auth] but it implements [serde::Serialize]
+/// To be removed once upstream equivalent is implementing Serialize (json serialization format
+/// should be the same), see [rust-bitcoincore-rpc/pull/181](https://github.com/rust-bitcoin/rust-bitcoincore-rpc/pull/181)
+#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(untagged)]
+pub enum Auth {
+    /// None authentication
+    None,
+    /// Authentication with username and password, usually [Auth::Cookie] should be preferred
+    UserPass {
+        /// Username
+        username: String,
+        /// Password
+        password: String,
+    },
+    /// Authentication with a cookie file
+    Cookie {
+        /// Cookie file
+        file: PathBuf,
+    },
+}
+
+impl From<Auth> for RpcAuth {
+    fn from(auth: Auth) -> Self {
+        match auth {
+            Auth::None => RpcAuth::None,
+            Auth::UserPass { username, password } => RpcAuth::UserPass(username, password),
+            Auth::Cookie { file } => RpcAuth::CookieFile(file),
+        }
+    }
+}
+
+impl RpcBlockchain {
+    fn get_node_synced_height(&self) -> Result<u32, Error> {
+        let info = self.client.get_address_info(&self._storage_address)?;
+        if let Some(GetAddressInfoResultLabel::Simple(label)) = info.labels.first() {
+            Ok(label
+                .parse::<u32>()
+                .unwrap_or_else(|_| self.skip_blocks.unwrap_or(0)))
+        } else {
+            Ok(self.skip_blocks.unwrap_or(0))
+        }
+    }
+
+    /// Set the synced height in the core node by using a label of a fixed address so that
+    /// another client with the same descriptor doesn't rescan the blockchain
+    fn set_node_synced_height(&self, height: u32) -> Result<(), Error> {
+        Ok(self
+            .client
+            .set_label(&self._storage_address, &height.to_string())?)
+    }
+}
+
+impl Blockchain for RpcBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        self.capabilities.clone()
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        let mut scripts_pubkeys = database.iter_script_pubkeys(Some(KeychainKind::External))?;
+        scripts_pubkeys.extend(database.iter_script_pubkeys(Some(KeychainKind::Internal))?);
+        debug!(
+            "importing {} script_pubkeys (some maybe already imported)",
+            scripts_pubkeys.len()
+        );
+        let requests: Vec<_> = scripts_pubkeys
+            .iter()
+            .map(|s| ImportMultiRequest {
+                timestamp: ImportMultiRescanSince::Timestamp(0),
+                script_pubkey: Some(ImportMultiRequestScriptPubkey::Script(s)),
+                watchonly: Some(true),
+                ..Default::default()
+            })
+            .collect();
+        let options = ImportMultiOptions {
+            rescan: Some(false),
+        };
+        // Note we use import_multi because as of bitcoin core 0.21.0 many descriptors are not supported
+        // https://bitcoindevkit.org/descriptors/#compatibility-matrix
+        //TODO maybe convenient using import_descriptor for compatible descriptor and import_multi as fallback
+        self.client.import_multi(&requests, Some(&options))?;
+
+        loop {
+            let current_height = self.get_height()?;
+
+            // min because block invalidate may cause height to go down
+            let node_synced = self.get_node_synced_height()?.min(current_height);
+
+            let sync_up_to = node_synced.saturating_add(10_000).min(current_height);
+
+            debug!("rescan_blockchain from:{} to:{}", node_synced, sync_up_to);
+            self.client
+                .rescan_blockchain(Some(node_synced as usize), Some(sync_up_to as usize))?;
+            progress_update.update((sync_up_to as f32) / (current_height as f32), None)?;
+
+            self.set_node_synced_height(sync_up_to)?;
+
+            if sync_up_to == current_height {
+                break;
+            }
+        }
+
+        self.sync(database, progress_update)
+    }
+
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        db: &mut D,
+        _progress_update: P,
+    ) -> Result<(), Error> {
+        let mut indexes = HashMap::new();
+        for keykind in &[KeychainKind::External, KeychainKind::Internal] {
+            indexes.insert(*keykind, db.get_last_index(*keykind)?.unwrap_or(0));
+        }
+
+        let mut known_txs: HashMap<_, _> = db
+            .iter_txs(true)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let known_utxos: HashSet<_> = db.iter_utxos()?.into_iter().collect();
+
+        //TODO list_since_blocks would be more efficient
+        let current_utxo = self
+            .client
+            .list_unspent(Some(0), None, None, Some(true), None)?;
+        debug!("current_utxo len {}", current_utxo.len());
+
+        //TODO supported up to 1_000 txs, should use since_blocks or do paging
+        let list_txs = self
+            .client
+            .list_transactions(None, Some(1_000), None, Some(true))?;
+        let mut list_txs_ids = HashSet::new();
+
+        for tx_result in list_txs.iter().filter(|t| {
+            // list_txs returns all conflicting tx we want to
+            // filter out replaced tx => unconfirmed and not in the mempool
+            t.info.confirmations > 0 || self.client.get_mempool_entry(&t.info.txid).is_ok()
+        }) {
+            let txid = tx_result.info.txid;
+            list_txs_ids.insert(txid);
+            if let Some(mut known_tx) = known_txs.get_mut(&txid) {
+                let confirmation_time =
+                    ConfirmationTime::new(tx_result.info.blockheight, tx_result.info.blocktime);
+                if confirmation_time != known_tx.confirmation_time {
+                    // reorg may change tx height
+                    debug!(
+                        "updating tx({}) confirmation time to: {:?}",
+                        txid, confirmation_time
+                    );
+                    known_tx.confirmation_time = confirmation_time;
+                    db.set_tx(known_tx)?;
+                }
+            } else {
+                //TODO check there is already the raw tx in db?
+                let tx_result = self.client.get_transaction(&txid, Some(true))?;
+                let tx: Transaction = deserialize(&tx_result.hex)?;
+                let mut received = 0u64;
+                let mut sent = 0u64;
+                for output in tx.output.iter() {
+                    if let Ok(Some((kind, index))) =
+                        db.get_path_from_script_pubkey(&output.script_pubkey)
+                    {
+                        if index > *indexes.get(&kind).unwrap() {
+                            indexes.insert(kind, index);
+                        }
+                        received += output.value;
+                    }
+                }
+
+                for input in tx.input.iter() {
+                    if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+                        sent += previous_output.value;
+                    }
+                }
+
+                let td = TransactionDetails {
+                    transaction: Some(tx),
+                    txid: tx_result.info.txid,
+                    confirmation_time: ConfirmationTime::new(
+                        tx_result.info.blockheight,
+                        tx_result.info.blocktime,
+                    ),
+                    received,
+                    sent,
+                    fee: tx_result.fee.map(|f| f.as_sat().abs() as u64),
+                    verified: true,
+                };
+                debug!(
+                    "saving tx: {} tx_result.fee:{:?} td.fees:{:?}",
+                    td.txid, tx_result.fee, td.fee
+                );
+                db.set_tx(&td)?;
+            }
+        }
+
+        for known_txid in known_txs.keys() {
+            if !list_txs_ids.contains(known_txid) {
+                debug!("removing tx: {}", known_txid);
+                db.del_tx(known_txid, false)?;
+            }
+        }
+
+        let current_utxos: HashSet<_> = current_utxo
+            .into_iter()
+            .map(|u| {
+                Ok(LocalUtxo {
+                    outpoint: OutPoint::new(u.txid, u.vout),
+                    keychain: db
+                        .get_path_from_script_pubkey(&u.script_pub_key)?
+                        .ok_or(Error::TransactionNotFound)?
+                        .0,
+                    txout: TxOut {
+                        value: u.amount.as_sat(),
+                        script_pubkey: u.script_pub_key,
+                    },
+                })
+            })
+            .collect::<Result<_, Error>>()?;
+
+        let spent: HashSet<_> = known_utxos.difference(&current_utxos).collect();
+        for s in spent {
+            debug!("removing utxo: {:?}", s);
+            db.del_utxo(&s.outpoint)?;
+        }
+        let received: HashSet<_> = current_utxos.difference(&known_utxos).collect();
+        for s in received {
+            debug!("adding utxo: {:?}", s);
+            db.set_utxo(s)?;
+        }
+
+        for (keykind, index) in indexes {
+            debug!("{:?} max {}", keykind, index);
+            db.set_last_index(keykind, index)?;
+        }
+
+        Ok(())
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(Some(self.client.get_raw_transaction(txid, None)?))
+    }
+
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        Ok(self.client.send_raw_transaction(tx).map(|_| ())?)
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(self.client.get_blockchain_info().map(|i| i.blocks as u32)?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        let sat_per_kb = self
+            .client
+            .estimate_smart_fee(target as u16, None)?
+            .fee_rate
+            .ok_or(Error::FeeRateUnavailable)?
+            .as_sat() as f64;
+
+        Ok(FeeRate::from_sat_per_vb((sat_per_kb / 1000f64) as f32))
+    }
+}
+
+impl ConfigurableBlockchain for RpcBlockchain {
+    type Config = RpcConfig;
+
+    /// Returns RpcBlockchain backend creating an RPC client to a specific wallet named as the descriptor's checksum
+    /// if it's the first time it creates the wallet in the node and upon return is granted the wallet is loaded
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let wallet_name = config.wallet_name.clone();
+        let wallet_url = format!("{}/wallet/{}", config.url, &wallet_name);
+        debug!("connecting to {} auth:{:?}", wallet_url, config.auth);
+
+        let client = Client::new(wallet_url.as_str(), config.auth.clone().into())?;
+        let loaded_wallets = client.list_wallets()?;
+        if loaded_wallets.contains(&wallet_name) {
+            debug!("wallet already loaded {:?}", wallet_name);
+        } else {
+            let existing_wallets = list_wallet_dir(&client)?;
+            if existing_wallets.contains(&wallet_name) {
+                client.load_wallet(&wallet_name)?;
+                debug!("wallet loaded {:?}", wallet_name);
+            } else {
+                client.create_wallet(&wallet_name, Some(true), None, None, None)?;
+                debug!("wallet created {:?}", wallet_name);
+            }
+        }
+
+        let blockchain_info = client.get_blockchain_info()?;
+        let network = match blockchain_info.chain.as_str() {
+            "main" => Network::Bitcoin,
+            "test" => Network::Testnet,
+            "regtest" => Network::Regtest,
+            "signet" => Network::Signet,
+            _ => return Err(Error::Generic("Invalid network".to_string())),
+        };
+        if network != config.network {
+            return Err(Error::InvalidNetwork {
+                requested: config.network,
+                found: network,
+            });
+        }
+
+        let mut capabilities: HashSet<_> = vec![Capability::FullHistory].into_iter().collect();
+        let rpc_version = client.version()?;
+        if rpc_version >= 210_000 {
+            let info: HashMap<String, Value> = client.call("getindexinfo", &[]).unwrap();
+            if info.contains_key("txindex") {
+                capabilities.insert(Capability::GetAnyTx);
+                capabilities.insert(Capability::AccurateFees);
+            }
+        }
+
+        // this is just a fixed address used only to store a label containing the synced height in the node
+        let mut storage_address =
+            Address::from_str("bc1qst0rewf0wm4kw6qn6kv0e5tc56nkf9yhcxlhqv").unwrap();
+        storage_address.network = network;
+
+        Ok(RpcBlockchain {
+            client,
+            network,
+            capabilities,
+            _storage_address: storage_address,
+            skip_blocks: config.skip_blocks,
+        })
+    }
+}
+
+/// Deterministically generate a unique name given the descriptors defining the wallet
+pub fn wallet_name_from_descriptor<T>(
+    descriptor: T,
+    change_descriptor: Option<T>,
+    network: Network,
+    secp: &SecpCtx,
+) -> Result<String, Error>
+where
+    T: IntoWalletDescriptor,
+{
+    //TODO check descriptors contains only public keys
+    let descriptor = descriptor
+        .into_wallet_descriptor(secp, network)?
+        .0
+        .to_string();
+    let mut wallet_name = get_checksum(&descriptor[..descriptor.find('#').unwrap()])?;
+    if let Some(change_descriptor) = change_descriptor {
+        let change_descriptor = change_descriptor
+            .into_wallet_descriptor(secp, network)?
+            .0
+            .to_string();
+        wallet_name.push_str(
+            get_checksum(&change_descriptor[..change_descriptor.find('#').unwrap()])?.as_str(),
+        );
+    }
+
+    Ok(wallet_name)
+}
+
+/// return the wallets available in default wallet directory
+//TODO use bitcoincore_rpc method when PR #179 lands
+fn list_wallet_dir(client: &Client) -> Result<Vec<String>, Error> {
+    #[derive(Deserialize)]
+    struct Name {
+        name: String,
+    }
+    #[derive(Deserialize)]
+    struct CallResult {
+        wallets: Vec<Name>,
+    }
+
+    let result: CallResult = client.call("listwalletdir", &[])?;
+    Ok(result.wallets.into_iter().map(|n| n.name).collect())
+}
+
+#[cfg(test)]
+#[cfg(feature = "test-rpc")]
+crate::bdk_blockchain_tests! {
+
+    fn test_instance(test_client: &TestClient) -> RpcBlockchain {
+        let config = RpcConfig {
+            url: test_client.bitcoind.rpc_url(),
+            auth: Auth::Cookie { file: test_client.bitcoind.params.cookie_file.clone() },
+            network: Network::Regtest,
+            wallet_name: format!("client-wallet-test-{:?}", std::time::SystemTime::now() ),
+            skip_blocks: None,
+        };
+        RpcBlockchain::from_config(&config).unwrap()
+    }
+}

src/blockchain/utils.rs

@@ -21,7 +21,7 @@ use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{KeychainKind, LocalUtxo, TransactionDetails};
+use crate::types::{ConfirmationTime, KeychainKind, LocalUtxo, TransactionDetails};
 use crate::wallet::time::Instant;
 use crate::wallet::utils::ChunksIterator;
 
@@ -53,7 +53,7 @@ pub trait ElectrumLikeSync {
 
     fn electrum_like_setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
+        stop_gap: usize,
         db: &mut D,
         _progress_update: P,
     ) -> Result<(), Error> {
@@ -61,7 +61,6 @@ pub trait ElectrumLikeSync {
         let start = Instant::new();
         debug!("start setup");
 
-        let stop_gap = stop_gap.unwrap_or(20);
         let chunk_size = stop_gap;
 
         let mut history_txs_id = HashSet::new();
@@ -147,18 +146,19 @@ pub trait ElectrumLikeSync {
         // save any tx details not in db but in history_txs_id or with different height/timestamp
         for txid in history_txs_id.iter() {
             let height = txid_height.get(txid).cloned().flatten();
-            let timestamp = *new_timestamps.get(txid).unwrap_or(&0u64);
+            let timestamp = new_timestamps.get(txid).cloned();
             if let Some(tx_details) = txs_details_in_db.get(txid) {
-                // check if height matches, otherwise updates it
-                if tx_details.height != height {
+                // check if tx height matches, otherwise updates it. timestamp is not in the if clause
+                // because we are not asking headers for confirmed tx we know about
+                if tx_details.confirmation_time.as_ref().map(|c| c.height) != height {
+                    let confirmation_time = ConfirmationTime::new(height, timestamp);
                     let mut new_tx_details = tx_details.clone();
-                    new_tx_details.height = height;
-                    new_tx_details.timestamp = timestamp;
+                    new_tx_details.confirmation_time = confirmation_time;
                     batch.set_tx(&new_tx_details)?;
                 }
             } else {
                 save_transaction_details_and_utxos(
-                    &txid,
+                    txid,
                     db,
                     timestamp,
                     height,
@@ -171,7 +171,7 @@ pub trait ElectrumLikeSync {
         // remove any tx details in db but not in history_txs_id
         for txid in txs_details_in_db.keys() {
             if !history_txs_id.contains(txid) {
-                batch.del_tx(&txid, false)?;
+                batch.del_tx(txid, false)?;
             }
         }
 
@@ -238,9 +238,13 @@ pub trait ElectrumLikeSync {
         chunk_size: usize,
     ) -> Result<HashMap<Txid, u64>, Error> {
         let mut txid_timestamp = HashMap::new();
+        let txid_in_db_with_conf: HashSet<_> = txs_details_in_db
+            .values()
+            .filter_map(|details| details.confirmation_time.as_ref().map(|_| details.txid))
+            .collect();
         let needed_txid_height: HashMap<&Txid, u32> = txid_height
             .iter()
-            .filter(|(t, _)| txs_details_in_db.get(*t).is_none())
+            .filter(|(t, _)| !txid_in_db_with_conf.contains(*t))
             .filter_map(|(t, o)| o.map(|h| (t, h)))
             .collect();
         let needed_heights: HashSet<u32> = needed_txid_height.values().cloned().collect();
@@ -292,7 +296,7 @@ pub trait ElectrumLikeSync {
 fn save_transaction_details_and_utxos<D: BatchDatabase>(
     txid: &Txid,
     db: &mut D,
-    timestamp: u64,
+    timestamp: Option<u64>,
     height: Option<u32>,
     updates: &mut dyn BatchOperations,
     utxo_deps: &HashMap<OutPoint, OutPoint>,
@@ -329,7 +333,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
 
         // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
         if let Some(outpoint) = utxo_deps.get(&input.previous_output) {
-            updates.del_utxo(&outpoint)?;
+            updates.del_utxo(outpoint)?;
         }
     }
 
@@ -355,9 +359,9 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
         transaction: Some(tx),
         received: incoming,
         sent: outgoing,
-        height,
-        timestamp,
-        fees: inputs_sum.saturating_sub(outputs_sum), /* if the tx is a coinbase, fees would be negative */
+        confirmation_time: ConfirmationTime::new(height, timestamp),
+        fee: Some(inputs_sum.saturating_sub(outputs_sum)), /* if the tx is a coinbase, fees would be negative */
+        verified: height.is_some(),
     };
     updates.set_tx(&tx_details)?;
 
@@ -365,7 +369,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
 }
 
 /// returns utxo dependency as the inputs needed for the utxo to exist
-/// `tx_raw_in_db` must contains utxo's generating txs or errors witt [crate::Error::TransactionNotFound]
+/// `tx_raw_in_db` must contains utxo's generating txs or errors with [crate::Error::TransactionNotFound]
 fn utxos_deps<D: BatchDatabase>(
     db: &mut D,
     tx_raw_in_db: &HashMap<Txid, Transaction>,

src/database/any.rs

@@ -65,6 +65,8 @@ macro_rules! impl_inner_method {
             $enum_name::Memory(inner) => inner.$name( $($args, )* ),
             #[cfg(feature = "key-value-db")]
             $enum_name::Sled(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "sqlite")]
+            $enum_name::Sqlite(inner) => inner.$name( $($args, )* ),
         }
     }
 }
@@ -82,10 +84,15 @@ pub enum AnyDatabase {
     #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
     /// Simple key-value embedded database based on [`sled`]
     Sled(sled::Tree),
+    #[cfg(feature = "sqlite")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))]
+    /// Sqlite embedded database using [`rusqlite`]
+    Sqlite(sqlite::SqliteDatabase),
 }
 
 impl_from!(memory::MemoryDatabase, AnyDatabase, Memory,);
 impl_from!(sled::Tree, AnyDatabase, Sled, #[cfg(feature = "key-value-db")]);
+impl_from!(sqlite::SqliteDatabase, AnyDatabase, Sqlite, #[cfg(feature = "sqlite")]);
 
 /// Type that contains any of the [`BatchDatabase::Batch`] types defined by the library
 pub enum AnyBatch {
@@ -95,6 +102,10 @@ pub enum AnyBatch {
     #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
     /// Simple key-value embedded database based on [`sled`]
     Sled(<sled::Tree as BatchDatabase>::Batch),
+    #[cfg(feature = "sqlite")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))]
+    /// Sqlite embedded database using [`rusqlite`]
+    Sqlite(<sqlite::SqliteDatabase as BatchDatabase>::Batch),
 }
 
 impl_from!(
@@ -103,6 +114,7 @@ impl_from!(
     Memory,
 );
 impl_from!(<sled::Tree as BatchDatabase>::Batch, AnyBatch, Sled, #[cfg(feature = "key-value-db")]);
+impl_from!(<sqlite::SqliteDatabase as BatchDatabase>::Batch, AnyBatch, Sqlite, #[cfg(feature = "sqlite")]);
 
 impl BatchOperations for AnyDatabase {
     fn set_script_pubkey(
@@ -233,6 +245,10 @@ impl Database for AnyDatabase {
     fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
         impl_inner_method!(AnyDatabase, self, increment_last_index, keychain)
     }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, flush)
+    }
 }
 
 impl BatchOperations for AnyBatch {
@@ -296,19 +312,26 @@ impl BatchDatabase for AnyDatabase {
             AnyDatabase::Memory(inner) => inner.begin_batch().into(),
             #[cfg(feature = "key-value-db")]
             AnyDatabase::Sled(inner) => inner.begin_batch().into(),
+            #[cfg(feature = "sqlite")]
+            AnyDatabase::Sqlite(inner) => inner.begin_batch().into(),
         }
     }
     fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> {
         match self {
             AnyDatabase::Memory(db) => match batch {
                 AnyBatch::Memory(batch) => db.commit_batch(batch),
-                #[cfg(feature = "key-value-db")]
-                _ => unimplemented!("Sled batch shouldn't be used with Memory db."),
+                #[cfg(any(feature = "key-value-db", feature = "sqlite"))]
+                _ => unimplemented!("Other batch shouldn't be used with Memory db."),
             },
             #[cfg(feature = "key-value-db")]
             AnyDatabase::Sled(db) => match batch {
                 AnyBatch::Sled(batch) => db.commit_batch(batch),
-                _ => unimplemented!("Memory batch shouldn't be used with Sled db."),
+                _ => unimplemented!("Other batch shouldn't be used with Sled db."),
+            },
+            #[cfg(feature = "sqlite")]
+            AnyDatabase::Sqlite(db) => match batch {
+                AnyBatch::Sqlite(batch) => db.commit_batch(batch),
+                _ => unimplemented!("Other batch shouldn't be used with Sqlite db."),
             },
         }
     }
@@ -333,6 +356,23 @@ impl ConfigurableDatabase for sled::Tree {
     }
 }
 
+/// Configuration type for a [`sqlite::SqliteDatabase`] database
+#[cfg(feature = "sqlite")]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct SqliteDbConfiguration {
+    /// Main directory of the db
+    pub path: String,
+}
+
+#[cfg(feature = "sqlite")]
+impl ConfigurableDatabase for sqlite::SqliteDatabase {
+    type Config = SqliteDbConfiguration;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(sqlite::SqliteDatabase::new(config.path.clone()))
+    }
+}
+
 /// Type that can contain any of the database configurations defined by the library
 ///
 /// This allows storing a single configuration that can be loaded into an [`AnyDatabase`]
@@ -346,6 +386,10 @@ pub enum AnyDatabaseConfig {
     #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
     /// Simple key-value embedded database based on [`sled`]
     Sled(SledDbConfiguration),
+    #[cfg(feature = "sqlite")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))]
+    /// Sqlite embedded database using [`rusqlite`]
+    Sqlite(SqliteDbConfiguration),
 }
 
 impl ConfigurableDatabase for AnyDatabase {
@@ -358,9 +402,14 @@ impl ConfigurableDatabase for AnyDatabase {
             }
             #[cfg(feature = "key-value-db")]
             AnyDatabaseConfig::Sled(inner) => AnyDatabase::Sled(sled::Tree::from_config(inner)?),
+            #[cfg(feature = "sqlite")]
+            AnyDatabaseConfig::Sqlite(inner) => {
+                AnyDatabase::Sqlite(sqlite::SqliteDatabase::from_config(inner)?)
+            }
         })
     }
 }
 
 impl_from!((), AnyDatabaseConfig, Memory,);
 impl_from!(SledDbConfiguration, AnyDatabaseConfig, Sled, #[cfg(feature = "key-value-db")]);
+impl_from!(SqliteDbConfiguration, AnyDatabaseConfig, Sqlite, #[cfg(feature = "sqlite")]);

src/database/keyvalue.rs

@@ -320,7 +320,7 @@ impl Database for Tree {
             .map(|b| -> Result<_, Error> {
                 let mut txdetails: TransactionDetails = serde_json::from_slice(&b)?;
                 if include_raw {
-                    txdetails.transaction = self.get_raw_tx(&txid)?;
+                    txdetails.transaction = self.get_raw_tx(txid)?;
                 }
 
                 Ok(txdetails)
@@ -367,6 +367,10 @@ impl Database for Tree {
             Ok(val)
         })
     }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        Ok(Tree::flush(self).map(|_| ())?)
+    }
 }
 
 impl BatchDatabase for Tree {
@@ -383,6 +387,7 @@ impl BatchDatabase for Tree {
 
 #[cfg(test)]
 mod test {
+    use lazy_static::lazy_static;
     use std::sync::{Arc, Condvar, Mutex, Once};
     use std::time::{SystemTime, UNIX_EPOCH};
 

src/database/memory.rs

@@ -14,6 +14,7 @@
 //! This module defines an in-memory database type called [`MemoryDatabase`] that is based on a
 //! [`BTreeMap`].
 
+use std::any::Any;
 use std::collections::BTreeMap;
 use std::ops::Bound::{Excluded, Included};
 
@@ -110,7 +111,7 @@ fn after(key: &[u8]) -> Vec<u8> {
 /// [`database`]: crate::database
 #[derive(Debug, Default)]
 pub struct MemoryDatabase {
-    map: BTreeMap<Vec<u8>, Box<dyn std::any::Any>>,
+    map: BTreeMap<Vec<u8>, Box<dyn Any + Send + Sync>>,
     deleted_keys: Vec<Vec<u8>>,
 }
 
@@ -394,7 +395,7 @@ impl Database for MemoryDatabase {
         Ok(self.map.get(&key).map(|b| {
             let mut txdetails: TransactionDetails = b.downcast_ref().cloned().unwrap();
             if include_raw {
-                txdetails.transaction = self.get_raw_tx(&txid).unwrap();
+                txdetails.transaction = self.get_raw_tx(txid).unwrap();
             }
 
             txdetails
@@ -419,6 +420,10 @@ impl Database for MemoryDatabase {
 
         Ok(*value)
     }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        Ok(())
+    }
 }
 
 impl BatchDatabase for MemoryDatabase {
@@ -429,8 +434,8 @@ impl BatchDatabase for MemoryDatabase {
     }
 
     fn commit_batch(&mut self, mut batch: Self::Batch) -> Result<(), Error> {
-        for key in batch.deleted_keys {
-            self.map.remove(&key);
+        for key in batch.deleted_keys.iter() {
+            self.map.remove(key);
         }
         self.map.append(&mut batch.map);
         Ok(())
@@ -452,20 +457,21 @@ impl ConfigurableDatabase for MemoryDatabase {
 /// don't have `test` set.
 macro_rules! populate_test_db {
     ($db:expr, $tx_meta:expr, $current_height:expr$(,)?) => {{
+        use std::str::FromStr;
         use $crate::database::BatchOperations;
         let mut db = $db;
         let tx_meta = $tx_meta;
         let current_height: Option<u32> = $current_height;
-        let tx = Transaction {
+        let tx = $crate::bitcoin::Transaction {
             version: 1,
             lock_time: 0,
             input: vec![],
             output: tx_meta
                 .output
                 .iter()
-                .map(|out_meta| bitcoin::TxOut {
+                .map(|out_meta| $crate::bitcoin::TxOut {
                     value: out_meta.value,
-                    script_pubkey: bitcoin::Address::from_str(&out_meta.to_address)
+                    script_pubkey: $crate::bitcoin::Address::from_str(&out_meta.to_address)
                         .unwrap()
                         .script_pubkey(),
                 })
@@ -473,29 +479,32 @@ macro_rules! populate_test_db {
         };
 
         let txid = tx.txid();
-        let height = tx_meta
+        let confirmation_time = tx_meta
             .min_confirmations
-            .map(|conf| current_height.unwrap().checked_sub(conf as u32).unwrap());
+            .map(|conf| $crate::ConfirmationTime {
+                height: current_height.unwrap().checked_sub(conf as u32).unwrap(),
+                timestamp: 0,
+            });
 
-        let tx_details = TransactionDetails {
+        let tx_details = $crate::TransactionDetails {
             transaction: Some(tx.clone()),
             txid,
-            timestamp: 0,
-            height,
+            fee: Some(0),
             received: 0,
             sent: 0,
-            fees: 0,
+            confirmation_time,
+            verified: current_height.is_some(),
         };
 
         db.set_tx(&tx_details).unwrap();
         for (vout, out) in tx.output.iter().enumerate() {
-            db.set_utxo(&LocalUtxo {
+            db.set_utxo(&$crate::LocalUtxo {
                 txout: out.clone(),
-                outpoint: OutPoint {
+                outpoint: $crate::bitcoin::OutPoint {
                     txid,
                     vout: vout as u32,
                 },
-                keychain: KeychainKind::External,
+                keychain: $crate::KeychainKind::External,
             })
             .unwrap();
         }

src/database/mod.rs

@@ -36,6 +36,11 @@ pub use any::{AnyDatabase, AnyDatabaseConfig};
 #[cfg(feature = "key-value-db")]
 pub(crate) mod keyvalue;
 
+#[cfg(feature = "sqlite")]
+pub(crate) mod sqlite;
+#[cfg(feature = "sqlite")]
+pub use sqlite::SqliteDatabase;
+
 pub mod memory;
 pub use memory::MemoryDatabase;
 
@@ -134,6 +139,9 @@ pub trait Database: BatchOperations {
     ///
     /// It should insert and return `0` if not present in the database
     fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error>;
+
+    /// Force changes to be written to disk
+    fn flush(&mut self) -> Result<(), Error>;
 }
 
 /// Trait for a database that supports batch operations
@@ -314,11 +322,14 @@ pub mod test {
         let mut tx_details = TransactionDetails {
             transaction: Some(tx),
             txid,
-            timestamp: 123456,
             received: 1337,
             sent: 420420,
-            fees: 140,
-            height: Some(1000),
+            fee: Some(140),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 123456,
+                height: 1000,
+            }),
+            verified: true,
         };
 
         tree.set_tx(&tx_details).unwrap();

src/database/sqlite.rs

@@ -0,0 +1,968 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::hash_types::Txid;
+use bitcoin::{OutPoint, Script, Transaction, TxOut};
+
+use crate::database::{BatchDatabase, BatchOperations, Database};
+use crate::error::Error;
+use crate::types::*;
+
+use rusqlite::{named_params, Connection};
+
+static MIGRATIONS: &[&str] = &[
+    "CREATE TABLE version (version INTEGER)",
+    "INSERT INTO version VALUES (1)",
+    "CREATE TABLE script_pubkeys (keychain TEXT, child INTEGER, script BLOB);",
+    "CREATE INDEX idx_keychain_child ON script_pubkeys(keychain, child);",
+    "CREATE INDEX idx_script ON script_pubkeys(script);",
+    "CREATE TABLE utxos (value INTEGER, keychain TEXT, vout INTEGER, txid BLOB, script BLOB);",
+    "CREATE INDEX idx_txid_vout ON utxos(txid, vout);",
+    "CREATE TABLE transactions (txid BLOB, raw_tx BLOB);",
+    "CREATE INDEX idx_txid ON transactions(txid);",
+    "CREATE TABLE transaction_details (txid BLOB, timestamp INTEGER, received INTEGER, sent INTEGER, fee INTEGER, height INTEGER, verified INTEGER DEFAULT 0);",
+    "CREATE INDEX idx_txdetails_txid ON transaction_details(txid);",
+    "CREATE TABLE last_derivation_indices (keychain TEXT, value INTEGER);",
+    "CREATE UNIQUE INDEX idx_indices_keychain ON last_derivation_indices(keychain);",
+    "CREATE TABLE checksums (keychain TEXT, checksum BLOB);",
+    "CREATE INDEX idx_checksums_keychain ON checksums(keychain);",
+];
+
+/// Sqlite database stored on filesystem
+///
+/// This is a permanent storage solution for devices and platforms that provide a filesystem.
+/// [`crate::database`]
+#[derive(Debug)]
+pub struct SqliteDatabase {
+    /// Path on the local filesystem to store the sqlite file
+    pub path: String,
+    /// A rusqlite connection object to the sqlite database
+    pub connection: Connection,
+}
+
+impl SqliteDatabase {
+    /// Instantiate a new SqliteDatabase instance by creating a connection
+    /// to the database stored at path
+    pub fn new(path: String) -> Self {
+        let connection = get_connection(&path).unwrap();
+        SqliteDatabase { path, connection }
+    }
+    fn insert_script_pubkey(
+        &self,
+        keychain: String,
+        child: u32,
+        script: &[u8],
+    ) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached("INSERT INTO script_pubkeys (keychain, child, script) VALUES (:keychain, :child, :script)")?;
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":child": child,
+            ":script": script
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+    fn insert_utxo(
+        &self,
+        value: u64,
+        keychain: String,
+        vout: u32,
+        txid: &[u8],
+        script: &[u8],
+    ) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached("INSERT INTO utxos (value, keychain, vout, txid, script) VALUES (:value, :keychain, :vout, :txid, :script)")?;
+        statement.execute(named_params! {
+            ":value": value,
+            ":keychain": keychain,
+            ":vout": vout,
+            ":txid": txid,
+            ":script": script
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+    fn insert_transaction(&self, txid: &[u8], raw_tx: &[u8]) -> Result<i64, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("INSERT INTO transactions (txid, raw_tx) VALUES (:txid, :raw_tx)")?;
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":raw_tx": raw_tx,
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn update_transaction(&self, txid: &[u8], raw_tx: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("UPDATE transactions SET raw_tx=:raw_tx WHERE txid=:txid")?;
+
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":raw_tx": raw_tx,
+        })?;
+
+        Ok(())
+    }
+
+    fn insert_transaction_details(&self, transaction: &TransactionDetails) -> Result<i64, Error> {
+        let (timestamp, height) = match &transaction.confirmation_time {
+            Some(confirmation_time) => (
+                Some(confirmation_time.timestamp),
+                Some(confirmation_time.height),
+            ),
+            None => (None, None),
+        };
+
+        let txid: &[u8] = &transaction.txid;
+
+        let mut statement = self.connection.prepare_cached("INSERT INTO transaction_details (txid, timestamp, received, sent, fee, height, verified) VALUES (:txid, :timestamp, :received, :sent, :fee, :height, :verified)")?;
+
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":timestamp": timestamp,
+            ":received": transaction.received,
+            ":sent": transaction.sent,
+            ":fee": transaction.fee,
+            ":height": height,
+            ":verified": transaction.verified
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn update_transaction_details(&self, transaction: &TransactionDetails) -> Result<(), Error> {
+        let (timestamp, height) = match &transaction.confirmation_time {
+            Some(confirmation_time) => (
+                Some(confirmation_time.timestamp),
+                Some(confirmation_time.height),
+            ),
+            None => (None, None),
+        };
+
+        let txid: &[u8] = &transaction.txid;
+
+        let mut statement = self.connection.prepare_cached("UPDATE transaction_details SET timestamp=:timestamp, received=:received, sent=:sent, fee=:fee, height=:height, verified=:verified WHERE txid=:txid")?;
+
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":timestamp": timestamp,
+            ":received": transaction.received,
+            ":sent": transaction.sent,
+            ":fee": transaction.fee,
+            ":height": height,
+            ":verified": transaction.verified,
+        })?;
+
+        Ok(())
+    }
+
+    fn insert_last_derivation_index(&self, keychain: String, value: u32) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "INSERT INTO last_derivation_indices (keychain, value) VALUES (:keychain, :value)",
+        )?;
+
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":value": value,
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn insert_checksum(&self, keychain: String, checksum: &[u8]) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "INSERT INTO checksums (keychain, checksum) VALUES (:keychain, :checksum)",
+        )?;
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":checksum": checksum,
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn update_last_derivation_index(&self, keychain: String, value: u32) -> Result<(), Error> {
+        let mut statement = self.connection.prepare_cached(
+            "INSERT INTO last_derivation_indices (keychain, value) VALUES (:keychain, :value) ON CONFLICT(keychain) DO UPDATE SET value=:value WHERE keychain=:keychain",
+        )?;
+
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":value": value,
+        })?;
+
+        Ok(())
+    }
+
+    fn select_script_pubkeys(&self) -> Result<Vec<Script>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT script FROM script_pubkeys")?;
+        let mut scripts: Vec<Script> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let raw_script: Vec<u8> = row.get(0)?;
+            scripts.push(raw_script.into());
+        }
+
+        Ok(scripts)
+    }
+
+    fn select_script_pubkeys_by_keychain(&self, keychain: String) -> Result<Vec<Script>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT script FROM script_pubkeys WHERE keychain=:keychain")?;
+        let mut scripts: Vec<Script> = vec![];
+        let mut rows = statement.query(named_params! {":keychain": keychain})?;
+        while let Some(row) = rows.next()? {
+            let raw_script: Vec<u8> = row.get(0)?;
+            scripts.push(raw_script.into());
+        }
+
+        Ok(scripts)
+    }
+
+    fn select_script_pubkey_by_path(
+        &self,
+        keychain: String,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "SELECT script FROM script_pubkeys WHERE keychain=:keychain AND child=:child",
+        )?;
+        let mut rows = statement.query(named_params! {":keychain": keychain,":child": child})?;
+
+        match rows.next()? {
+            Some(row) => {
+                let script: Vec<u8> = row.get(0)?;
+                let script: Script = script.into();
+                Ok(Some(script))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_script_pubkey_by_script(
+        &self,
+        script: &[u8],
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT keychain, child FROM script_pubkeys WHERE script=:script")?;
+        let mut rows = statement.query(named_params! {":script": script})?;
+        match rows.next()? {
+            Some(row) => {
+                let keychain: String = row.get(0)?;
+                let keychain: KeychainKind = serde_json::from_str(&keychain)?;
+                let child: u32 = row.get(1)?;
+                Ok(Some((keychain, child)))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT value, keychain, vout, txid, script FROM utxos")?;
+        let mut utxos: Vec<LocalUtxo> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let value = row.get(0)?;
+            let keychain: String = row.get(1)?;
+            let vout = row.get(2)?;
+            let txid: Vec<u8> = row.get(3)?;
+            let script: Vec<u8> = row.get(4)?;
+
+            let keychain: KeychainKind = serde_json::from_str(&keychain)?;
+
+            utxos.push(LocalUtxo {
+                outpoint: OutPoint::new(deserialize(&txid)?, vout),
+                txout: TxOut {
+                    value,
+                    script_pubkey: script.into(),
+                },
+                keychain,
+            })
+        }
+
+        Ok(utxos)
+    }
+
+    fn select_utxo_by_outpoint(
+        &self,
+        txid: &[u8],
+        vout: u32,
+    ) -> Result<Option<(u64, KeychainKind, Script)>, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "SELECT value, keychain, script FROM utxos WHERE txid=:txid AND vout=:vout",
+        )?;
+        let mut rows = statement.query(named_params! {":txid": txid,":vout": vout})?;
+        match rows.next()? {
+            Some(row) => {
+                let value: u64 = row.get(0)?;
+                let keychain: String = row.get(1)?;
+                let keychain: KeychainKind = serde_json::from_str(&keychain)?;
+                let script: Vec<u8> = row.get(2)?;
+                let script: Script = script.into();
+
+                Ok(Some((value, keychain, script)))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_transactions(&self) -> Result<Vec<Transaction>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT raw_tx FROM transactions")?;
+        let mut txs: Vec<Transaction> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let raw_tx: Vec<u8> = row.get(0)?;
+            let tx: Transaction = deserialize(&raw_tx)?;
+            txs.push(tx);
+        }
+        Ok(txs)
+    }
+
+    fn select_transaction_by_txid(&self, txid: &[u8]) -> Result<Option<Transaction>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT raw_tx FROM transactions WHERE txid=:txid")?;
+        let mut rows = statement.query(named_params! {":txid": txid})?;
+        match rows.next()? {
+            Some(row) => {
+                let raw_tx: Vec<u8> = row.get(0)?;
+                let tx: Transaction = deserialize(&raw_tx)?;
+                Ok(Some(tx))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_transaction_details_with_raw(&self) -> Result<Vec<TransactionDetails>, Error> {
+        let mut statement = self.connection.prepare_cached("SELECT transaction_details.txid, transaction_details.timestamp, transaction_details.received, transaction_details.sent, transaction_details.fee, transaction_details.height, transaction_details.verified, transactions.raw_tx FROM transaction_details, transactions WHERE transaction_details.txid = transactions.txid")?;
+        let mut transaction_details: Vec<TransactionDetails> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let txid: Vec<u8> = row.get(0)?;
+            let txid: Txid = deserialize(&txid)?;
+            let timestamp: Option<u64> = row.get(1)?;
+            let received: u64 = row.get(2)?;
+            let sent: u64 = row.get(3)?;
+            let fee: Option<u64> = row.get(4)?;
+            let height: Option<u32> = row.get(5)?;
+            let verified: bool = row.get(6)?;
+            let raw_tx: Option<Vec<u8>> = row.get(7)?;
+            let tx: Option<Transaction> = match raw_tx {
+                Some(raw_tx) => {
+                    let tx: Transaction = deserialize(&raw_tx)?;
+                    Some(tx)
+                }
+                None => None,
+            };
+
+            let confirmation_time = match (height, timestamp) {
+                (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+                _ => None,
+            };
+
+            transaction_details.push(TransactionDetails {
+                transaction: tx,
+                txid,
+                received,
+                sent,
+                fee,
+                confirmation_time,
+                verified,
+            });
+        }
+        Ok(transaction_details)
+    }
+
+    fn select_transaction_details(&self) -> Result<Vec<TransactionDetails>, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "SELECT txid, timestamp, received, sent, fee, height, verified FROM transaction_details",
+        )?;
+        let mut transaction_details: Vec<TransactionDetails> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let txid: Vec<u8> = row.get(0)?;
+            let txid: Txid = deserialize(&txid)?;
+            let timestamp: Option<u64> = row.get(1)?;
+            let received: u64 = row.get(2)?;
+            let sent: u64 = row.get(3)?;
+            let fee: Option<u64> = row.get(4)?;
+            let height: Option<u32> = row.get(5)?;
+            let verified: bool = row.get(6)?;
+
+            let confirmation_time = match (height, timestamp) {
+                (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+                _ => None,
+            };
+
+            transaction_details.push(TransactionDetails {
+                transaction: None,
+                txid,
+                received,
+                sent,
+                fee,
+                confirmation_time,
+                verified,
+            });
+        }
+        Ok(transaction_details)
+    }
+
+    fn select_transaction_details_by_txid(
+        &self,
+        txid: &[u8],
+    ) -> Result<Option<TransactionDetails>, Error> {
+        let mut statement = self.connection.prepare_cached("SELECT transaction_details.timestamp, transaction_details.received, transaction_details.sent, transaction_details.fee, transaction_details.height, transaction_details.verified, transactions.raw_tx FROM transaction_details, transactions WHERE transaction_details.txid=transactions.txid AND transaction_details.txid=:txid")?;
+        let mut rows = statement.query(named_params! { ":txid": txid })?;
+
+        match rows.next()? {
+            Some(row) => {
+                let timestamp: Option<u64> = row.get(0)?;
+                let received: u64 = row.get(1)?;
+                let sent: u64 = row.get(2)?;
+                let fee: Option<u64> = row.get(3)?;
+                let height: Option<u32> = row.get(4)?;
+                let verified: bool = row.get(5)?;
+
+                let raw_tx: Option<Vec<u8>> = row.get(6)?;
+                let tx: Option<Transaction> = match raw_tx {
+                    Some(raw_tx) => {
+                        let tx: Transaction = deserialize(&raw_tx)?;
+                        Some(tx)
+                    }
+                    None => None,
+                };
+
+                let confirmation_time = match (height, timestamp) {
+                    (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+                    _ => None,
+                };
+
+                Ok(Some(TransactionDetails {
+                    transaction: tx,
+                    txid: deserialize(txid)?,
+                    received,
+                    sent,
+                    fee,
+                    confirmation_time,
+                    verified,
+                }))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_last_derivation_index_by_keychain(
+        &self,
+        keychain: String,
+    ) -> Result<Option<u32>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT value FROM last_derivation_indices WHERE keychain=:keychain")?;
+        let mut rows = statement.query(named_params! {":keychain": keychain})?;
+        match rows.next()? {
+            Some(row) => {
+                let value: u32 = row.get(0)?;
+                Ok(Some(value))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_checksum_by_keychain(&self, keychain: String) -> Result<Option<Vec<u8>>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT checksum FROM checksums WHERE keychain=:keychain")?;
+        let mut rows = statement.query(named_params! {":keychain": keychain})?;
+
+        match rows.next()? {
+            Some(row) => {
+                let checksum: Vec<u8> = row.get(0)?;
+                Ok(Some(checksum))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn delete_script_pubkey_by_path(&self, keychain: String, child: u32) -> Result<(), Error> {
+        let mut statement = self.connection.prepare_cached(
+            "DELETE FROM script_pubkeys WHERE keychain=:keychain AND child=:child",
+        )?;
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":child": child
+        })?;
+
+        Ok(())
+    }
+
+    fn delete_script_pubkey_by_script(&self, script: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM script_pubkeys WHERE script=:script")?;
+        statement.execute(named_params! {
+            ":script": script
+        })?;
+
+        Ok(())
+    }
+
+    fn delete_utxo_by_outpoint(&self, txid: &[u8], vout: u32) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM utxos WHERE txid=:txid AND vout=:vout")?;
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":vout": vout
+        })?;
+
+        Ok(())
+    }
+
+    fn delete_transaction_by_txid(&self, txid: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM transactions WHERE txid=:txid")?;
+        statement.execute(named_params! {":txid": txid})?;
+        Ok(())
+    }
+
+    fn delete_transaction_details_by_txid(&self, txid: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM transaction_details WHERE txid=:txid")?;
+        statement.execute(named_params! {":txid": txid})?;
+        Ok(())
+    }
+
+    fn delete_last_derivation_index_by_keychain(&self, keychain: String) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM last_derivation_indices WHERE keychain=:keychain")?;
+        statement.execute(named_params! {
+            ":keychain": &keychain
+        })?;
+
+        Ok(())
+    }
+}
+
+impl BatchOperations for SqliteDatabase {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        self.insert_script_pubkey(keychain, child, script.as_bytes())?;
+        Ok(())
+    }
+
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+        self.insert_utxo(
+            utxo.txout.value,
+            serde_json::to_string(&utxo.keychain)?,
+            utxo.outpoint.vout,
+            &utxo.outpoint.txid,
+            utxo.txout.script_pubkey.as_bytes(),
+        )?;
+        Ok(())
+    }
+
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        match self.select_transaction_by_txid(&transaction.txid())? {
+            Some(_) => {
+                self.update_transaction(&transaction.txid(), &serialize(transaction))?;
+            }
+            None => {
+                self.insert_transaction(&transaction.txid(), &serialize(transaction))?;
+            }
+        }
+        Ok(())
+    }
+
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        match self.select_transaction_details_by_txid(&transaction.txid)? {
+            Some(_) => {
+                self.update_transaction_details(transaction)?;
+            }
+            None => {
+                self.insert_transaction_details(transaction)?;
+            }
+        }
+
+        if let Some(tx) = &transaction.transaction {
+            self.set_raw_tx(tx)?;
+        }
+
+        Ok(())
+    }
+
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        self.update_last_derivation_index(serde_json::to_string(&keychain)?, value)?;
+        Ok(())
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        let script = self.select_script_pubkey_by_path(keychain.clone(), child)?;
+        match script {
+            Some(script) => {
+                self.delete_script_pubkey_by_path(keychain, child)?;
+                Ok(Some(script))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        match self.select_script_pubkey_by_script(script.as_bytes())? {
+            Some((keychain, child)) => {
+                self.delete_script_pubkey_by_script(script.as_bytes())?;
+                Ok(Some((keychain, child)))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        match self.select_utxo_by_outpoint(&outpoint.txid, outpoint.vout)? {
+            Some((value, keychain, script_pubkey)) => {
+                self.delete_utxo_by_outpoint(&outpoint.txid, outpoint.vout)?;
+                Ok(Some(LocalUtxo {
+                    outpoint: *outpoint,
+                    txout: TxOut {
+                        value,
+                        script_pubkey,
+                    },
+                    keychain,
+                }))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        match self.select_transaction_by_txid(txid)? {
+            Some(tx) => {
+                self.delete_transaction_by_txid(txid)?;
+                Ok(Some(tx))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        match self.select_transaction_details_by_txid(txid)? {
+            Some(transaction_details) => {
+                self.delete_transaction_details_by_txid(txid)?;
+
+                if include_raw {
+                    self.delete_transaction_by_txid(txid)?;
+                }
+                Ok(Some(transaction_details))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        match self.select_last_derivation_index_by_keychain(keychain.clone())? {
+            Some(value) => {
+                self.delete_last_derivation_index_by_keychain(keychain)?;
+
+                Ok(Some(value))
+            }
+            None => Ok(None),
+        }
+    }
+}
+
+impl Database for SqliteDatabase {
+    fn check_descriptor_checksum<B: AsRef<[u8]>>(
+        &mut self,
+        keychain: KeychainKind,
+        bytes: B,
+    ) -> Result<(), Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+
+        match self.select_checksum_by_keychain(keychain.clone())? {
+            Some(checksum) => {
+                if checksum == bytes.as_ref().to_vec() {
+                    Ok(())
+                } else {
+                    Err(Error::ChecksumMismatch)
+                }
+            }
+            None => {
+                self.insert_checksum(keychain, bytes.as_ref())?;
+                Ok(())
+            }
+        }
+    }
+
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        match keychain {
+            Some(keychain) => {
+                let keychain = serde_json::to_string(&keychain)?;
+                self.select_script_pubkeys_by_keychain(keychain)
+            }
+            None => self.select_script_pubkeys(),
+        }
+    }
+
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        self.select_utxos()
+    }
+
+    fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error> {
+        self.select_transactions()
+    }
+
+    fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error> {
+        match include_raw {
+            true => self.select_transaction_details_with_raw(),
+            false => self.select_transaction_details(),
+        }
+    }
+
+    fn get_script_pubkey_from_path(
+        &self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        match self.select_script_pubkey_by_path(keychain, child)? {
+            Some(script) => Ok(Some(script)),
+            None => Ok(None),
+        }
+    }
+
+    fn get_path_from_script_pubkey(
+        &self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        match self.select_script_pubkey_by_script(script.as_bytes())? {
+            Some((keychain, child)) => Ok(Some((keychain, child))),
+            None => Ok(None),
+        }
+    }
+
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        match self.select_utxo_by_outpoint(&outpoint.txid, outpoint.vout)? {
+            Some((value, keychain, script_pubkey)) => Ok(Some(LocalUtxo {
+                outpoint: *outpoint,
+                txout: TxOut {
+                    value,
+                    script_pubkey,
+                },
+                keychain,
+            })),
+            None => Ok(None),
+        }
+    }
+
+    fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        match self.select_transaction_by_txid(txid)? {
+            Some(tx) => Ok(Some(tx)),
+            None => Ok(None),
+        }
+    }
+
+    fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error> {
+        match self.select_transaction_details_by_txid(txid)? {
+            Some(mut transaction_details) => {
+                if !include_raw {
+                    transaction_details.transaction = None;
+                }
+                Ok(Some(transaction_details))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        let value = self.select_last_derivation_index_by_keychain(keychain)?;
+        Ok(value)
+    }
+
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let keychain_string = serde_json::to_string(&keychain)?;
+        match self.get_last_index(keychain)? {
+            Some(value) => {
+                self.update_last_derivation_index(keychain_string, value + 1)?;
+                Ok(value + 1)
+            }
+            None => {
+                self.insert_last_derivation_index(keychain_string, 0)?;
+                Ok(0)
+            }
+        }
+    }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
+impl BatchDatabase for SqliteDatabase {
+    type Batch = SqliteDatabase;
+
+    fn begin_batch(&self) -> Self::Batch {
+        let db = SqliteDatabase::new(self.path.clone());
+        db.connection.execute("BEGIN TRANSACTION", []).unwrap();
+        db
+    }
+
+    fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> {
+        batch.connection.execute("COMMIT TRANSACTION", [])?;
+        Ok(())
+    }
+}
+
+pub fn get_connection(path: &str) -> Result<Connection, Error> {
+    let connection = Connection::open(path)?;
+    migrate(&connection)?;
+    Ok(connection)
+}
+
+pub fn get_schema_version(conn: &Connection) -> rusqlite::Result<i32> {
+    let statement = conn.prepare_cached("SELECT version FROM version");
+    match statement {
+        Err(rusqlite::Error::SqliteFailure(e, Some(msg))) => {
+            if msg == "no such table: version" {
+                Ok(0)
+            } else {
+                Err(rusqlite::Error::SqliteFailure(e, Some(msg)))
+            }
+        }
+        Ok(mut stmt) => {
+            let mut rows = stmt.query([])?;
+            match rows.next()? {
+                Some(row) => {
+                    let version: i32 = row.get(0)?;
+                    Ok(version)
+                }
+                None => Ok(0),
+            }
+        }
+        _ => Ok(0),
+    }
+}
+
+pub fn set_schema_version(conn: &Connection, version: i32) -> rusqlite::Result<usize> {
+    conn.execute(
+        "UPDATE version SET version=:version",
+        named_params! {":version": version},
+    )
+}
+
+pub fn migrate(conn: &Connection) -> rusqlite::Result<()> {
+    let version = get_schema_version(conn)?;
+    let stmts = &MIGRATIONS[(version as usize)..];
+    let mut i: i32 = version;
+
+    if version == MIGRATIONS.len() as i32 {
+        log::info!("db up to date, no migration needed");
+        return Ok(());
+    }
+
+    for stmt in stmts {
+        let res = conn.execute(stmt, []);
+        if res.is_err() {
+            println!("migration failed on:\n{}\n{:?}", stmt, res);
+            break;
+        }
+
+        i += 1;
+    }
+
+    set_schema_version(conn, i)?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+pub mod test {
+    use crate::database::SqliteDatabase;
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn get_database() -> SqliteDatabase {
+        let time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+        let mut dir = std::env::temp_dir();
+        dir.push(format!("bdk_{}", time.as_nanos()));
+        SqliteDatabase::new(String::from(dir.to_str().unwrap()))
+    }
+
+    #[test]
+    fn test_script_pubkey() {
+        crate::database::test::test_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_batch_script_pubkey() {
+        crate::database::test::test_batch_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_iter_script_pubkey() {
+        crate::database::test::test_iter_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_del_script_pubkey() {
+        crate::database::test::test_del_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_utxo() {
+        crate::database::test::test_utxo(get_database());
+    }
+
+    #[test]
+    fn test_raw_tx() {
+        crate::database::test::test_raw_tx(get_database());
+    }
+
+    #[test]
+    fn test_tx() {
+        crate::database::test::test_tx(get_database());
+    }
+
+    #[test]
+    fn test_last_index() {
+        crate::database::test::test_last_index(get_database());
+    }
+}

src/descriptor/dsl.rs

@@ -175,7 +175,7 @@ macro_rules! impl_node_opcode_two {
 #[doc(hidden)]
 #[macro_export]
 macro_rules! impl_node_opcode_three {
-    ( $terminal_variant:ident, $( $inner:tt )* ) => {
+    ( $terminal_variant:ident, $( $inner:tt )* ) => ({
         use $crate::descriptor::CheckMiniscript;
 
         let inner = $crate::fragment_internal!( @t $( $inner )* );
@@ -201,7 +201,7 @@ macro_rules! impl_node_opcode_three {
 
                 Ok((minisc, a_keymap, networks))
             })
-    };
+    });
 }
 
 #[doc(hidden)]
@@ -571,8 +571,9 @@ macro_rules! fragment {
     ( pk ( $key:expr ) ) => ({
         $crate::fragment!(c:pk_k ( $key ))
     });
-    ( pk_h ( $key_hash:expr ) ) => ({
-        $crate::impl_leaf_opcode_value!(PkH, $key_hash)
+    ( pk_h ( $key:expr ) ) => ({
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+        $crate::keys::make_pkh($key, &secp)
     });
     ( after ( $value:expr ) ) => ({
         $crate::impl_leaf_opcode_value!(After, $value)
@@ -601,6 +602,9 @@ macro_rules! fragment {
     ( and_or ( $( $inner:tt )* ) ) => ({
         $crate::impl_node_opcode_three!(AndOr, $( $inner )*)
     });
+    ( andor ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_three!(AndOr, $( $inner )*)
+    });
     ( or_b ( $( $inner:tt )* ) ) => ({
         $crate::impl_node_opcode_two!(OrB, $( $inner )*)
     });
@@ -790,6 +794,25 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_fixed_threeop_descriptors() {
+        let redeem_key = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        let move_key = bitcoin::PublicKey::from_str(
+            "032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af",
+        )
+        .unwrap();
+
+        check(
+            descriptor!(sh(wsh(and_or(pk(redeem_key), older(1000), pk(move_key))))),
+            true,
+            true,
+            &["2MypGwr5eQWAWWJtiJgUEToVxc4zuokjQRe"],
+        );
+    }
+
     #[test]
     fn test_bip32_legacy_descriptors() {
         let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();

src/descriptor/mod.rs

@@ -128,11 +128,11 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
             let (pk, _, networks) = if self.0.is_witness() {
                 let desciptor_key: DescriptorKey<miniscript::Segwitv0> =
                     pk.clone().into_descriptor_key()?;
-                desciptor_key.extract(&secp)?
+                desciptor_key.extract(secp)?
             } else {
                 let desciptor_key: DescriptorKey<miniscript::Legacy> =
                     pk.clone().into_descriptor_key()?;
-                desciptor_key.extract(&secp)?
+                desciptor_key.extract(secp)?
             };
 
             if networks.contains(&network) {

src/descriptor/policy.rs

@@ -47,14 +47,12 @@ use bitcoin::util::bip32::Fingerprint;
 use bitcoin::PublicKey;
 
 use miniscript::descriptor::{DescriptorPublicKey, ShInner, SortedMultiVec, WshInner};
-use miniscript::{
-    Descriptor, Miniscript, MiniscriptKey, Satisfier, ScriptContext, Terminal, ToPublicKey,
-};
+use miniscript::{Descriptor, Miniscript, MiniscriptKey, Satisfier, ScriptContext, Terminal};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use crate::descriptor::{DerivedDescriptorKey, ExtractPolicy};
+use crate::descriptor::ExtractPolicy;
 use crate::wallet::signer::{SignerId, SignersContainer};
 use crate::wallet::utils::{self, After, Older, SecpCtx};
 
@@ -88,13 +86,6 @@ impl PkOrF {
             },
         }
     }
-
-    fn from_key_hash(k: hash160::Hash) -> Self {
-        PkOrF {
-            pubkey_hash: Some(k),
-            ..Default::default()
-        }
-    }
 }
 
 /// An item that needs to be satisfied
@@ -336,7 +327,7 @@ impl Satisfaction {
                         items.push(inner_index);
                         let conditions_set = other_conditions
                             .values()
-                            .fold(HashSet::new(), |set, i| set.union(&i).cloned().collect());
+                            .fold(HashSet::new(), |set, i| set.union(i).cloned().collect());
                         conditions.insert(inner_index, conditions_set);
                     }
                 }
@@ -779,25 +770,6 @@ fn signature_in_psbt(psbt: &Psbt, key: &DescriptorPublicKey, secp: &SecpCtx) ->
     })
 }
 
-fn signature_key(
-    key: &<DescriptorPublicKey as MiniscriptKey>::Hash,
-    signers: &SignersContainer,
-    secp: &SecpCtx,
-) -> Policy {
-    let key_hash = DerivedDescriptorKey::new(key.clone(), secp)
-        .to_public_key()
-        .to_pubkeyhash();
-    let mut policy: Policy = SatisfiableItem::Signature(PkOrF::from_key_hash(key_hash)).into();
-
-    if signers.find(SignerId::PkHash(key_hash)).is_some() {
-        policy.contribution = Satisfaction::Complete {
-            condition: Default::default(),
-        }
-    }
-
-    policy
-}
-
 impl<Ctx: ScriptContext> ExtractPolicy for Miniscript<DescriptorPublicKey, Ctx> {
     fn extract_policy(
         &self,
@@ -809,7 +781,7 @@ impl<Ctx: ScriptContext> ExtractPolicy for Miniscript<DescriptorPublicKey, Ctx>
             // Leaves
             Terminal::True | Terminal::False => None,
             Terminal::PkK(pubkey) => Some(signature(pubkey, signers, build_sat, secp)),
-            Terminal::PkH(pubkey_hash) => Some(signature_key(pubkey_hash, signers, secp)),
+            Terminal::PkH(pubkey_hash) => Some(signature(pubkey_hash, signers, build_sat, secp)),
             Terminal::After(value) => {
                 let mut policy: Policy = SatisfiableItem::AbsoluteTimelock { value: *value }.into();
                 policy.contribution = Satisfaction::Complete {
@@ -1007,7 +979,6 @@ mod test {
     use crate::descriptor::{ExtractPolicy, IntoWalletDescriptor};
 
     use super::*;
-    use crate::bitcoin::consensus::deserialize;
     use crate::descriptor::derived::AsDerived;
     use crate::descriptor::policy::SatisfiableItem::{Multisig, Signature, Thresh};
     use crate::keys::{DescriptorKey, IntoDescriptorKey};
@@ -1031,8 +1002,8 @@ mod test {
     ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
         let path = bip32::DerivationPath::from_str(path).unwrap();
         let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
-        let tpub = bip32::ExtendedPubKey::from_private(&secp, &tprv);
-        let fingerprint = tprv.fingerprint(&secp);
+        let tpub = bip32::ExtendedPubKey::from_private(secp, &tprv);
+        let fingerprint = tprv.fingerprint(secp);
         let prvkey = (tprv, path.clone()).into_descriptor_key().unwrap();
         let pubkey = (tpub, path).into_descriptor_key().unwrap();
 
@@ -1445,6 +1416,7 @@ mod test {
 
     const ALICE_TPRV_STR:&str = "tprv8ZgxMBicQKsPf6T5X327efHnvJDr45Xnb8W4JifNWtEoqXu9MRYS4v1oYe6DFcMVETxy5w3bqpubYRqvcVTqovG1LifFcVUuJcbwJwrhYzP";
     const BOB_TPRV_STR:&str = "tprv8ZgxMBicQKsPeinZ155cJAn117KYhbaN6MV3WeG6sWhxWzcvX1eg1awd4C9GpUN1ncLEM2rzEvunAg3GizdZD4QPPCkisTz99tXXB4wZArp";
+    const CAROL_TPRV_STR:&str = "tprv8ZgxMBicQKsPdC3CicFifuLCEyVVdXVUNYorxUWj3iGZ6nimnLAYAY9SYB7ib8rKzRxrCKFcEytCt6szwd2GHnGPRCBLAEAoSVDefSNk4Bt";
     const ALICE_BOB_PATH: &str = "m/0'";
 
     #[test]
@@ -1475,7 +1447,7 @@ mod test {
 
         let signers_container = Arc::new(SignersContainer::from(keymap));
 
-        let psbt: Psbt = deserialize(&base64::decode(ALICE_SIGNED_PSBT).unwrap()).unwrap();
+        let psbt = Psbt::from_str(ALICE_SIGNED_PSBT).unwrap();
 
         let policy_alice_psbt = wallet_desc
             .extract_policy(&signers_container, BuildSatisfaction::Psbt(&psbt), &secp)
@@ -1490,7 +1462,7 @@ mod test {
             )
         );
 
-        let psbt: Psbt = deserialize(&base64::decode(BOB_SIGNED_PSBT).unwrap()).unwrap();
+        let psbt = Psbt::from_str(BOB_SIGNED_PSBT).unwrap();
         let policy_bob_psbt = wallet_desc
             .extract_policy(&signers_container, BuildSatisfaction::Psbt(&psbt), &secp)
             .unwrap()
@@ -1504,7 +1476,7 @@ mod test {
             )
         );
 
-        let psbt: Psbt = deserialize(&base64::decode(ALICE_BOB_SIGNED_PSBT).unwrap()).unwrap();
+        let psbt = Psbt::from_str(ALICE_BOB_SIGNED_PSBT).unwrap();
         let policy_alice_bob_psbt = wallet_desc
             .extract_policy(&signers_container, BuildSatisfaction::Psbt(&psbt), &secp)
             .unwrap()
@@ -1545,8 +1517,7 @@ mod test {
             addr.to_string()
         );
 
-        let psbt: Psbt =
-            deserialize(&base64::decode(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED).unwrap()).unwrap();
+        let psbt = Psbt::from_str(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED).unwrap();
 
         let build_sat = BuildSatisfaction::PsbtTimelocks {
             psbt: &psbt,
@@ -1584,9 +1555,7 @@ mod test {
         );
         //println!("{}", serde_json::to_string(&policy_expired).unwrap());
 
-        let psbt_signed: Psbt =
-            deserialize(&base64::decode(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED_SIGNED).unwrap())
-                .unwrap();
+        let psbt_signed = Psbt::from_str(PSBT_POLICY_CONSIDER_TIMELOCK_EXPIRED_SIGNED).unwrap();
 
         let build_sat_expired_signed = BuildSatisfaction::PsbtTimelocks {
             psbt: &psbt_signed,
@@ -1606,4 +1575,28 @@ mod test {
         );
         //println!("{}", serde_json::to_string(&policy_expired_signed).unwrap());
     }
+
+    #[test]
+    fn test_extract_pkh() {
+        let secp = Secp256k1::new();
+
+        let (prvkey_alice, _, _) = setup_keys(ALICE_TPRV_STR, ALICE_BOB_PATH, &secp);
+        let (prvkey_bob, _, _) = setup_keys(BOB_TPRV_STR, ALICE_BOB_PATH, &secp);
+        let (prvkey_carol, _, _) = setup_keys(CAROL_TPRV_STR, ALICE_BOB_PATH, &secp);
+
+        let desc = descriptor!(wsh(c: andor(
+            pk(prvkey_alice),
+            pk_k(prvkey_bob),
+            pk_h(prvkey_carol),
+        )))
+        .unwrap();
+
+        let (wallet_desc, keymap) = desc
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .unwrap();
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+
+        let policy = wallet_desc.extract_policy(&signers_container, BuildSatisfaction::None, &secp);
+        assert!(policy.is_ok());
+    }
 }

src/error.rs

@@ -11,6 +11,7 @@
 
 use std::fmt;
 
+use crate::bitcoin::Network;
 use crate::{descriptor, wallet, wallet::address_validator};
 use bitcoin::OutPoint;
 
@@ -23,10 +24,6 @@ pub enum Error {
     Generic(String),
     /// This error is thrown when trying to convert Bare and Public key script to address
     ScriptDoesntHaveAddressForm,
-    /// Found multiple outputs when `single_recipient` option has been specified
-    SingleRecipientMultipleOutputs,
-    /// `single_recipient` option is selected but neither `drain_wallet` nor `manually_selected_only` are
-    SingleRecipientNoInputs,
     /// Cannot build a tx without recipients
     NoRecipients,
     /// `manually_selected_only` option is selected but no utxo has been passed
@@ -64,6 +61,8 @@ pub enum Error {
         /// Required fee absolute value (satoshi)
         required: u64,
     },
+    /// Node doesn't have data to estimate a fee rate
+    FeeRateUnavailable,
     /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
     /// key in the descriptor must either be a master key itself (having depth = 0) or have an
     /// explicit origin provided
@@ -80,6 +79,16 @@ pub enum Error {
     InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
     /// Signing error
     Signer(crate::wallet::signer::SignerError),
+    /// Invalid network
+    InvalidNetwork {
+        /// requested network, for example what is given as bdk-cli option
+        requested: Network,
+        /// found network, for example the network of the bitcoin node
+        found: Network,
+    },
+    #[cfg(feature = "verify")]
+    /// Transaction verification error
+    Verification(crate::wallet::verify::VerifyError),
 
     /// Progress value must be between `0.0` (included) and `100.0` (included)
     InvalidProgressValue(f32),
@@ -106,6 +115,8 @@ pub enum Error {
     Hex(bitcoin::hashes::hex::Error),
     /// Partially signed bitcoin transaction error
     Psbt(bitcoin::util::psbt::Error),
+    /// Partially signed bitcoin transaction parseerror
+    PsbtParse(bitcoin::util::psbt::PsbtParseError),
 
     //KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
     //MissingInputUTXO(usize),
@@ -119,13 +130,19 @@ pub enum Error {
     Electrum(electrum_client::Error),
     #[cfg(feature = "esplora")]
     /// Esplora client error
-    Esplora(crate::blockchain::esplora::EsploraError),
+    Esplora(Box<crate::blockchain::esplora::EsploraError>),
     #[cfg(feature = "compact_filters")]
     /// Compact filters client error)
     CompactFilters(crate::blockchain::compact_filters::CompactFiltersError),
     #[cfg(feature = "key-value-db")]
     /// Sled database error
     Sled(sled::Error),
+    #[cfg(feature = "rpc")]
+    /// Rpc client error
+    Rpc(bitcoincore_rpc::Error),
+    #[cfg(feature = "sqlite")]
+    /// Rusqlite client error
+    Rusqlite(rusqlite::Error),
 }
 
 impl fmt::Display for Error {
@@ -172,13 +189,16 @@ impl_error!(bitcoin::secp256k1::Error, Secp256k1);
 impl_error!(serde_json::Error, Json);
 impl_error!(bitcoin::hashes::hex::Error, Hex);
 impl_error!(bitcoin::util::psbt::Error, Psbt);
+impl_error!(bitcoin::util::psbt::PsbtParseError, PsbtParse);
 
 #[cfg(feature = "electrum")]
 impl_error!(electrum_client::Error, Electrum);
-#[cfg(feature = "esplora")]
-impl_error!(crate::blockchain::esplora::EsploraError, Esplora);
 #[cfg(feature = "key-value-db")]
 impl_error!(sled::Error, Sled);
+#[cfg(feature = "rpc")]
+impl_error!(bitcoincore_rpc::Error, Rpc);
+#[cfg(feature = "sqlite")]
+impl_error!(rusqlite::Error, Rusqlite);
 
 #[cfg(feature = "compact_filters")]
 impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
@@ -189,3 +209,20 @@ impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
         }
     }
 }
+
+#[cfg(feature = "verify")]
+impl From<crate::wallet::verify::VerifyError> for Error {
+    fn from(other: crate::wallet::verify::VerifyError) -> Self {
+        match other {
+            crate::wallet::verify::VerifyError::Global(inner) => *inner,
+            err => Error::Verification(err),
+        }
+    }
+}
+
+#[cfg(feature = "esplora")]
+impl From<crate::blockchain::esplora::EsploraError> for Error {
+    fn from(other: crate::blockchain::esplora::EsploraError) -> Self {
+        Error::Esplora(Box::new(other))
+    }
+}

src/keys/bip39.rs

@@ -40,7 +40,7 @@ pub type MnemonicWithPassphrase = (Mnemonic, Option<String>);
 #[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
 impl<Ctx: ScriptContext> DerivableKey<Ctx> for Seed {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-        Ok(bip32::ExtendedPrivKey::new_master(Network::Bitcoin, &self.as_bytes())?.into())
+        Ok(bip32::ExtendedPrivKey::new_master(Network::Bitcoin, self.as_bytes())?.into())
     }
 
     fn into_descriptor_key(

src/keys/mod.rs

@@ -753,6 +753,20 @@ pub fn make_pk<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((minisc, key_map, valid_networks))
 }
 
+// Used internally by `bdk::fragment!` to build `pk_h()` fragments
+#[doc(hidden)]
+pub fn make_pkh<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    descriptor_key: Pk,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), DescriptorError> {
+    let (key, key_map, valid_networks) = descriptor_key.into_descriptor_key()?.extract(secp)?;
+    let minisc = Miniscript::from_ast(Terminal::PkH(key))?;
+
+    minisc.check_minsicript()?;
+
+    Ok((minisc, key_map, valid_networks))
+}
+
 // Used internally by `bdk::fragment!` to build `multi()` fragments
 #[doc(hidden)]
 pub fn make_multi<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(

src/lib.rs

@@ -40,7 +40,7 @@
 //! interact with the bitcoin P2P network.
 //!
 //! ```toml
-//! bdk = "0.8.0"
+//! bdk = "0.13.0"
 //! ```
 #![cfg_attr(
     feature = "electrum",
@@ -104,8 +104,6 @@ fn main() -> Result<(), bdk::Error> {
 
 ### Example
 ```no_run
-use base64::decode;
-
 use bdk::{FeeRate, Wallet};
 use bdk::database::MemoryDatabase;
 use bdk::blockchain::{noop_progress, ElectrumBlockchain};
@@ -138,7 +136,7 @@ fn main() -> Result<(), bdk::Error> {
     };
 
     println!("Transaction details: {:#?}", details);
-    println!("Unsigned PSBT: {}", base64::encode(&serialize(&psbt)));
+    println!("Unsigned PSBT: {}", &psbt);
 
     Ok(())
 }
@@ -150,8 +148,9 @@ fn main() -> Result<(), bdk::Error> {
 //!
 //! ### Example
 //! ```no_run
-//! use base64::decode;
-//! use bitcoin::consensus::deserialize;
+//! use std::str::FromStr;
+//!
+//! use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
 //!
 //! use bdk::{Wallet, SignOptions};
 //! use bdk::database::MemoryDatabase;
@@ -165,7 +164,7 @@ fn main() -> Result<(), bdk::Error> {
 //!     )?;
 //!
 //!     let psbt = "...";
-//!     let mut psbt = deserialize(&base64::decode(psbt).unwrap())?;
+//!     let mut psbt = Psbt::from_str(psbt)?;
 //!
 //!     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
 //!
@@ -206,11 +205,24 @@ extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
+#[cfg(all(feature = "reqwest", feature = "ureq"))]
+compile_error!("Features reqwest and ureq are mutually exclusive and cannot be enabled together");
+
 #[cfg(all(feature = "async-interface", feature = "electrum"))]
 compile_error!(
     "Features async-interface and electrum are mutually exclusive and cannot be enabled together"
 );
 
+#[cfg(all(feature = "async-interface", feature = "ureq"))]
+compile_error!(
+    "Features async-interface and ureq are mutually exclusive and cannot be enabled together"
+);
+
+#[cfg(all(feature = "async-interface", feature = "compact_filters"))]
+compile_error!(
+    "Features async-interface and compact_filters are mutually exclusive and cannot be enabled together"
+);
+
 #[cfg(feature = "keys-bip39")]
 extern crate bip39;
 
@@ -223,22 +235,19 @@ extern crate bdk_macros;
 #[cfg(feature = "compact_filters")]
 extern crate lazy_static;
 
+#[cfg(feature = "rpc")]
+pub extern crate bitcoincore_rpc;
+
 #[cfg(feature = "electrum")]
 pub extern crate electrum_client;
 
-#[cfg(feature = "esplora")]
-pub extern crate reqwest;
-
 #[cfg(feature = "key-value-db")]
 pub extern crate sled;
 
-#[allow(unused_imports)]
-#[cfg(test)]
-#[allow(unused_imports)]
-#[cfg(test)]
-#[macro_use]
-pub extern crate serial_test;
+#[cfg(feature = "sqlite")]
+pub extern crate rusqlite;
 
+#[allow(unused_imports)]
 #[macro_use]
 pub(crate) mod error;
 pub mod blockchain;

src/psbt/mod.rs

@@ -41,12 +41,12 @@ impl PsbtUtils for Psbt {
 
 #[cfg(test)]
 mod test {
-    use crate::bitcoin::consensus::deserialize;
     use crate::bitcoin::TxIn;
     use crate::psbt::Psbt;
-    use crate::wallet::test::{get_funded_wallet, get_test_wpkh};
     use crate::wallet::AddressIndex;
+    use crate::wallet::{get_funded_wallet, test::get_test_wpkh};
     use crate::SignOptions;
+    use std::str::FromStr;
 
     // from bip 174
     const PSBT_STR: &str = "cHNidP8BAKACAAAAAqsJSaCMWvfEm4IS9Bfi8Vqz9cM9zxU4IagTn4d6W3vkAAAAAAD+////qwlJoIxa98SbghL0F+LxWrP1wz3PFTghqBOfh3pbe+QBAAAAAP7///8CYDvqCwAAAAAZdqkUdopAu9dAy+gdmI5x3ipNXHE5ax2IrI4kAAAAAAAAGXapFG9GILVT+glechue4O/p+gOcykWXiKwAAAAAAAEHakcwRAIgR1lmF5fAGwNrJZKJSGhiGDR9iYZLcZ4ff89X0eURZYcCIFMJ6r9Wqk2Ikf/REf3xM286KdqGbX+EhtdVRs7tr5MZASEDXNxh/HupccC1AaZGoqg7ECy0OIEhfKaC3Ibi1z+ogpIAAQEgAOH1BQAAAAAXqRQ1RebjO4MsRwUPJNPuuTycA5SLx4cBBBYAFIXRNTfy4mVAWjTbr6nj3aAfuCMIAAAA";
@@ -54,7 +54,7 @@ mod test {
     #[test]
     #[should_panic(expected = "InputIndexOutOfRange")]
     fn test_psbt_malformed_psbt_input_legacy() {
-        let psbt_bip: Psbt = deserialize(&base64::decode(PSBT_STR).unwrap()).unwrap();
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let send_to = wallet.get_address(AddressIndex::New).unwrap();
         let mut builder = wallet.build_tx();
@@ -71,7 +71,7 @@ mod test {
     #[test]
     #[should_panic(expected = "InputIndexOutOfRange")]
     fn test_psbt_malformed_psbt_input_segwit() {
-        let psbt_bip: Psbt = deserialize(&base64::decode(PSBT_STR).unwrap()).unwrap();
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let send_to = wallet.get_address(AddressIndex::New).unwrap();
         let mut builder = wallet.build_tx();
@@ -103,7 +103,7 @@ mod test {
 
     #[test]
     fn test_psbt_sign_with_finalized() {
-        let psbt_bip: Psbt = deserialize(&base64::decode(PSBT_STR).unwrap()).unwrap();
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
         let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
         let send_to = wallet.get_address(AddressIndex::New).unwrap();
         let mut builder = wallet.build_tx();

src/testutils/blockchain_tests.rs

@@ -6,38 +6,54 @@ use bitcoin::{Address, Amount, Script, Transaction, Txid};
 pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
 pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
 use core::str::FromStr;
+use electrsd::bitcoind::BitcoinD;
+use electrsd::{bitcoind, ElectrsD};
 pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
 #[allow(unused_imports)]
-use log::{debug, error, info, trace};
+use log::{debug, error, info, log_enabled, trace, Level};
 use std::collections::HashMap;
 use std::env;
 use std::ops::Deref;
-use std::path::PathBuf;
 use std::time::Duration;
 
 pub struct TestClient {
-    client: RpcClient,
-    electrum: ElectrumClient,
+    pub bitcoind: BitcoinD,
+    pub electrsd: ElectrsD,
 }
 
 impl TestClient {
-    pub fn new(rpc_host_and_wallet: String, rpc_wallet_name: String) -> Self {
-        let client = RpcClient::new(
-            format!("http://{}/wallet/{}", rpc_host_and_wallet, rpc_wallet_name),
-            get_auth(),
-        )
-        .unwrap();
-        let electrum = ElectrumClient::new(&get_electrum_url()).unwrap();
+    pub fn new(bitcoind_exe: String, electrs_exe: String) -> Self {
+        debug!("launching {} and {}", &bitcoind_exe, &electrs_exe);
 
-        TestClient { client, electrum }
+        let mut conf = bitcoind::Conf::default();
+        conf.view_stdout = log_enabled!(Level::Debug);
+        let bitcoind = BitcoinD::with_conf(bitcoind_exe, &conf).unwrap();
+
+        let mut conf = electrsd::Conf::default();
+        conf.view_stderr = log_enabled!(Level::Debug);
+        conf.http_enabled = cfg!(feature = "test-esplora");
+
+        let electrsd = ElectrsD::with_conf(electrs_exe, &bitcoind, &conf).unwrap();
+
+        let node_address = bitcoind.client.get_new_address(None, None).unwrap();
+        bitcoind
+            .client
+            .generate_to_address(101, &node_address)
+            .unwrap();
+
+        let mut test_client = TestClient { bitcoind, electrsd };
+        TestClient::wait_for_block(&mut test_client, 101);
+        test_client
     }
 
     fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
         // wait for electrs to index the tx
         exponential_backoff_poll(|| {
+            self.electrsd.trigger().unwrap();
             trace!("wait_for_tx {}", txid);
 
-            self.electrum
+            self.electrsd
+                .client
                 .script_get_history(monitor_script)
                 .unwrap()
                 .iter()
@@ -46,12 +62,13 @@ impl TestClient {
     }
 
     fn wait_for_block(&mut self, min_height: usize) {
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         loop {
             let header = exponential_backoff_poll(|| {
-                self.electrum.ping().unwrap();
-                self.electrum.block_headers_pop().unwrap()
+                self.electrsd.trigger().unwrap();
+                self.electrsd.client.ping().unwrap();
+                self.electrsd.client.block_headers_pop().unwrap()
             });
             if header.height >= min_height {
                 break;
@@ -96,10 +113,13 @@ impl TestClient {
             .unwrap();
 
         // broadcast through electrum so that it caches the tx immediately
+
         let txid = self
-            .electrum
+            .electrsd
+            .client
             .transaction_broadcast(&deserialize(&tx.hex).unwrap())
             .unwrap();
+        debug!("broadcasted to electrum {}", txid);
 
         if let Some(num) = meta_tx.min_confirmations {
             self.generate(num, None);
@@ -209,7 +229,7 @@ impl TestClient {
         let block_hex: String = serialize(&block).to_hex();
         debug!("generated block hex: {}", block_hex);
 
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         let submit_result: serde_json::Value =
             self.call("submitblock", &[block_hex.into()]).unwrap();
@@ -237,7 +257,7 @@ impl TestClient {
     }
 
     pub fn invalidate(&mut self, num_blocks: u64) {
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         let best_hash = self.get_best_block_hash().unwrap();
         let initial_height = self.get_block_info(&best_hash).unwrap().height;
@@ -288,16 +308,25 @@ impl Deref for TestClient {
     type Target = RpcClient;
 
     fn deref(&self) -> &Self::Target {
-        &self.client
+        &self.bitcoind.client
     }
 }
 
 impl Default for TestClient {
     fn default() -> Self {
-        let rpc_host_and_port =
-            env::var("BDK_RPC_URL").unwrap_or_else(|_| "127.0.0.1:18443".to_string());
-        let wallet = env::var("BDK_RPC_WALLET").unwrap_or_else(|_| "bdk-test".to_string());
-        Self::new(rpc_host_and_port, wallet)
+        let bitcoind_exe = env::var("BITCOIND_EXE")
+            .ok()
+            .or(bitcoind::downloaded_exe_path())
+            .expect(
+                "you should provide env var BITCOIND_EXE or specifiy a bitcoind version feature",
+            );
+        let electrs_exe = env::var("ELECTRS_EXE")
+            .ok()
+            .or(electrsd::downloaded_exe_path())
+            .expect(
+                "you should provide env var ELECTRS_EXE or specifiy a electrsd version feature",
+            );
+        Self::new(bitcoind_exe, electrs_exe)
     }
 }
 
@@ -317,27 +346,13 @@ where
     }
 }
 
-// TODO: we currently only support env vars, we could also parse a toml file
-fn get_auth() -> Auth {
-    match env::var("BDK_RPC_AUTH").as_ref().map(String::as_ref) {
-        Ok("USER_PASS") => Auth::UserPass(
-            env::var("BDK_RPC_USER").unwrap(),
-            env::var("BDK_RPC_PASS").unwrap(),
-        ),
-        _ => Auth::CookieFile(PathBuf::from(
-            env::var("BDK_RPC_COOKIEFILE")
-                .unwrap_or_else(|_| "/home/user/.bitcoin/regtest/.cookie".to_string()),
-        )),
-    }
-}
-
 /// This macro runs blockchain tests against a `Blockchain` implementation. It requires access to a
 /// Bitcoin core wallet via RPC. At the moment you have to dig into the code yourself and look at
 /// the setup required to run the tests yourself.
 #[macro_export]
 macro_rules! bdk_blockchain_tests {
     (
-     fn test_instance() -> $blockchain:ty $block:block) => {
+     fn $_fn_name:ident ( $( $test_client:ident : &TestClient )? $(,)? ) -> $blockchain:ty $block:block) => {
         #[cfg(test)]
         mod bdk_blockchain_tests {
             use $crate::bitcoin::Network;
@@ -346,18 +361,18 @@ macro_rules! bdk_blockchain_tests {
             use $crate::database::MemoryDatabase;
             use $crate::types::KeychainKind;
             use $crate::{Wallet, FeeRate};
-            use $crate::wallet::AddressIndex::New;
             use $crate::testutils;
-            use $crate::serial_test::serial;
 
             use super::*;
 
-            fn get_blockchain() -> $blockchain {
+            #[allow(unused_variables)]
+            fn get_blockchain(test_client: &TestClient) -> $blockchain {
+                $( let $test_client = test_client; )?
                 $block
             }
 
-            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<$blockchain, MemoryDatabase> {
-                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
+            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>), test_client: &TestClient) -> Wallet<$blockchain, MemoryDatabase> {
+                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain(test_client)).unwrap()
             }
 
             fn init_single_sig() -> (Wallet<$blockchain, MemoryDatabase>, (String, Option<String>), TestClient) {
@@ -368,13 +383,16 @@ macro_rules! bdk_blockchain_tests {
                 };
 
                 let test_client = TestClient::default();
-                let wallet = get_wallet_from_descriptors(&descriptors);
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
+
+                // rpc need to call import_multi before receiving any tx, otherwise will not see tx in the mempool
+                #[cfg(feature = "test-rpc")]
+                wallet.sync(noop_progress(), None).unwrap();
 
                 (wallet, descriptors, test_client)
             }
 
             #[test]
-            #[serial]
             fn test_sync_simple() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -386,18 +404,17 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External, "incorrect keychain kind");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.received, 50_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time");
             }
 
             #[test]
-            #[serial]
             fn test_sync_stop_gap_20() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -410,12 +427,11 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 100_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                assert_eq!(wallet.get_balance().unwrap(), 100_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
             }
 
             #[test]
-            #[serial]
             fn test_sync_before_and_after_receive() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -428,12 +444,11 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
             }
 
             #[test]
-            #[serial]
             fn test_sync_multiple_outputs_same_tx() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -443,19 +458,18 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 105_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 3);
+                assert_eq!(wallet.get_balance().unwrap(), 105_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 3, "incorrect number of unspents");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.received, 105_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 105_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_multi() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -468,13 +482,12 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 2);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 2, "incorrect number of unspent");
             }
 
             #[test]
-            #[serial]
             fn test_sync_address_reuse() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -490,11 +503,10 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_rbf_replaced() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -504,36 +516,35 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.received, 50_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
 
                 let new_txid = test_client.bump_fee(&txid);
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after bump");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs after bump");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent after bump");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, new_txid);
-                assert_eq!(list_tx_item.received, 50_000);
-                assert_eq!(list_tx_item.sent, 0);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, new_txid, "incorrect txid after bump");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received after bump");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent after bump");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect height after bump");
             }
 
             // FIXME: I would like this to be cfg_attr(not(feature = "test-esplora"), ignore) but it
             // doesn't work for some reason.
             #[cfg(not(feature = "esplora"))]
             #[test]
-            #[serial]
             fn test_sync_reorg_block() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -543,28 +554,27 @@ macro_rules! bdk_blockchain_tests {
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert!(list_tx_item.height.is_some());
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert!(list_tx_item.confirmation_time.is_some(), "incorrect confirmation_time");
 
                 // Invalidate 1 block
                 test_client.invalidate(1);
 
                 wallet.sync(noop_progress(), None).unwrap();
 
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after invalidate");
 
                 let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                assert_eq!(list_tx_item.txid, txid);
-                assert_eq!(list_tx_item.height, None);
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid after invalidate");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time after invalidate");
             }
 
             #[test]
-            #[serial]
             fn test_sync_after_send() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 println!("{}", descriptors.0);
@@ -575,7 +585,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey(), 25_000);
@@ -585,18 +595,17 @@ macro_rules! bdk_blockchain_tests {
                 let tx = psbt.extract_tx();
                 println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
                 wallet.broadcast(tx).unwrap();
-
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after send");
 
-                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
             }
 
             #[test]
-            #[serial]
-            fn test_sync_outgoing_from_scratch() {
+            fn test_update_confirmation_time_after_generate() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
+                println!("{}", descriptors.0);
                 let node_addr = test_client.get_node_address(None);
 
                 let received_txid = test_client.receive(testutils! {
@@ -604,36 +613,63 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_none());
+
+                test_client.generate(1, Some(node_addr));
+                wallet.sync(noop_progress(), None).unwrap();
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_some());
+
+            }
+
+            #[test]
+            fn test_sync_outgoing_from_scratch() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+                let received_txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey(), 25_000);
                 let (mut psbt, details) = builder.finish().unwrap();
+
                 let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
                 assert!(finalized, "Cannot finalize transaction");
                 let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after receive");
 
                 // empty wallet
-                let wallet = get_wallet_from_descriptors(&descriptors);
-                wallet.sync(noop_progress(), None).unwrap();
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
 
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
+                wallet.sync(noop_progress(), None).unwrap();
                 let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
 
                 let received = tx_map.get(&received_txid).unwrap();
-                assert_eq!(received.received, 50_000);
-                assert_eq!(received.sent, 0);
+                assert_eq!(received.received, 50_000, "incorrect received from receiver");
+                assert_eq!(received.sent, 0, "incorrect sent from receiver");
 
                 let sent = tx_map.get(&sent_txid).unwrap();
-                assert_eq!(sent.received, details.received);
-                assert_eq!(sent.sent, details.sent);
-                assert_eq!(sent.fees, details.fees);
+                assert_eq!(sent.received, details.received, "incorrect received from sender");
+                assert_eq!(sent.sent, details.sent, "incorrect sent from sender");
+                assert_eq!(sent.fee.unwrap_or(0), details.fee.unwrap_or(0), "incorrect fees from sender");
             }
 
             #[test]
-            #[serial]
             fn test_sync_long_change_chain() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -643,7 +679,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut total_sent = 0;
                 for _ in 0..5 {
@@ -656,21 +692,26 @@ macro_rules! bdk_blockchain_tests {
 
                     wallet.sync(noop_progress(), None).unwrap();
 
-                    total_sent += 5_000 + details.fees;
+                    total_sent += 5_000 + details.fee.unwrap_or(0);
                 }
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance after chain");
 
                 // empty wallet
-                let wallet = get_wallet_from_descriptors(&descriptors);
+
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
+
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance empty wallet");
+
             }
 
             #[test]
-            #[serial]
-            fn test_sync_bump_fee() {
+            fn test_sync_bump_fee_basic() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
 
@@ -679,7 +720,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 5_000).enable_rbf();
@@ -688,8 +729,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fees - 5_000);
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance from received");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(2.1));
@@ -698,14 +739,13 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fees - 5_000);
-                assert_eq!(wallet.get_balance().unwrap(), new_details.received);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees after bump");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance from received after bump");
 
-                assert!(new_details.fees > details.fees);
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_remove_change() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -715,7 +755,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
@@ -724,8 +764,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fees);
-                assert_eq!(wallet.get_balance().unwrap(), details.received);
+                assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect received after send");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(5.0));
@@ -734,15 +774,14 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 0);
-                assert_eq!(new_details.received, 0);
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after change removal");
+                assert_eq!(new_details.received, 0, "incorrect received after change removal");
 
-                assert!(new_details.fees > details.fees);
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
             }
 
             #[test]
-            #[serial]
-            fn test_sync_bump_fee_add_input() {
+            fn test_sync_bump_fee_add_input_simple() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
 
@@ -751,7 +790,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
@@ -760,8 +799,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                assert_eq!(details.received, 1_000 - details.fees);
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(10.0));
@@ -770,12 +809,11 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(new_details.sent, 75_000);
-                assert_eq!(wallet.get_balance().unwrap(), new_details.received);
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance after add input");
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_add_input_no_change() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -785,7 +823,7 @@ macro_rules! bdk_blockchain_tests {
                 });
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
 
                 let mut builder = wallet.build_tx();
                 builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
@@ -794,8 +832,8 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                assert_eq!(details.received, 1_000 - details.fees);
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
 
                 let mut builder = wallet.build_fee_bump(details.txid).unwrap();
                 builder.fee_rate(FeeRate::from_sat_per_vb(123.0));
@@ -806,25 +844,38 @@ macro_rules! bdk_blockchain_tests {
                 assert!(finalized, "Cannot finalize transaction");
                 wallet.broadcast(new_psbt.extract_tx()).unwrap();
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(new_details.sent, 75_000);
-                assert_eq!(wallet.get_balance().unwrap(), 0);
-                assert_eq!(new_details.received, 0);
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after add input");
+                assert_eq!(new_details.received, 0, "incorrect received after add input");
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_coinbase() {
                 let (wallet, _, mut test_client) = init_single_sig();
-                let wallet_addr = wallet.get_address(New).unwrap().address;
+
+                let wallet_addr = wallet.get_address($crate::wallet::AddressIndex::New).unwrap().address;
 
                 wallet.sync(noop_progress(), None).unwrap();
-                assert_eq!(wallet.get_balance().unwrap(), 0);
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance");
 
                 test_client.generate(1, Some(wallet_addr));
 
+                #[cfg(feature = "rpc")]
+                {
+                    // rpc consider coinbase only when mature (100 blocks)
+                    let node_addr = test_client.get_node_address(None);
+                    test_client.generate(100, Some(node_addr));
+                }
+
+
                 wallet.sync(noop_progress(), None).unwrap();
-                assert!(wallet.get_balance().unwrap() > 0);
+                assert!(wallet.get_balance().unwrap() > 0, "incorrect balance after receiving coinbase");
             }
         }
-    }
+    };
+
+    ( fn $fn_name:ident ($( $tt:tt )+) -> $blockchain:ty $block:block) => {
+        compile_error!(concat!("Invalid arguments `", stringify!($($tt)*), "` in the blockchain tests fn."));
+        compile_error!("Only the exact `&TestClient` type is supported, **without** any leading path items.");
+    };
 }

src/testutils/mod.rs

@@ -10,6 +10,7 @@
 // licenses.
 #![allow(missing_docs)]
 
+#[cfg(test)]
 #[cfg(feature = "test-blockchains")]
 pub mod blockchain_tests;
 
@@ -99,8 +100,8 @@ impl TranslateDescriptor for Descriptor<DescriptorPublicKey> {
 #[macro_export]
 macro_rules! testutils {
     ( @external $descriptors:expr, $child:expr ) => ({
-        use bitcoin::secp256k1::Secp256k1;
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
+        use $crate::bitcoin::secp256k1::Secp256k1;
+        use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
 
         use $crate::testutils::TranslateDescriptor;
 
@@ -110,15 +111,15 @@ macro_rules! testutils {
         parsed.derive_translated(&secp, $child).address(bitcoin::Network::Regtest).expect("No address form")
     });
     ( @internal $descriptors:expr, $child:expr ) => ({
-        use bitcoin::secp256k1::Secp256k1;
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
+        use $crate::bitcoin::secp256k1::Secp256k1;
+        use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
 
         use $crate::testutils::TranslateDescriptor;
 
         let secp = Secp256k1::new();
 
         let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
-        parsed.derive_translated(&secp, $child).address(bitcoin::Network::Regtest).expect("No address form")
+        parsed.derive_translated(&secp, $child).address($crate::bitcoin::Network::Regtest).expect("No address form")
     });
     ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
     ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
@@ -144,8 +145,8 @@ macro_rules! testutils {
         let mut seed = [0u8; 32];
         rand::thread_rng().fill(&mut seed[..]);
 
-        let key = bitcoin::util::bip32::ExtendedPrivKey::new_master(
-            bitcoin::Network::Testnet,
+        let key = $crate::bitcoin::util::bip32::ExtendedPrivKey::new_master(
+            $crate::bitcoin::Network::Testnet,
             &seed,
         );
 
@@ -157,13 +158,13 @@ macro_rules! testutils {
     ( @generate_wif ) => ({
         use rand::Rng;
 
-        let mut key = [0u8; bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
+        let mut key = [0u8; $crate::bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
         rand::thread_rng().fill(&mut key[..]);
 
-        (bitcoin::PrivateKey {
+        ($crate::bitcoin::PrivateKey {
             compressed: true,
-            network: bitcoin::Network::Testnet,
-            key: bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
+            network: $crate::bitcoin::Network::Testnet,
+            key: $crate::bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
         }.to_string(), None::<String>, None::<String>)
     });
 
@@ -180,8 +181,8 @@ macro_rules! testutils {
     ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )? $( ( @keys $( $keys:tt )* ) )* ) => ({
         use std::str::FromStr;
         use std::collections::HashMap;
-        use miniscript::descriptor::Descriptor;
-        use miniscript::TranslatePk;
+        use $crate::miniscript::descriptor::Descriptor;
+        use $crate::miniscript::TranslatePk;
 
         #[allow(unused_assignments, unused_mut)]
         let mut keys: HashMap<&'static str, (String, Option<String>, Option<String>)> = HashMap::new();

src/types.rs

@@ -10,6 +10,7 @@
 // licenses.
 
 use std::convert::AsRef;
+use std::ops::Sub;
 
 use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
 use bitcoin::{hash_types::Txid, util::psbt};
@@ -65,10 +66,31 @@ impl FeeRate {
         FeeRate(1.0)
     }
 
+    /// Calculate fee rate from `fee` and weight units (`wu`).
+    pub fn from_wu(fee: u64, wu: usize) -> FeeRate {
+        Self::from_vb(fee, wu.vbytes())
+    }
+
+    /// Calculate fee rate from `fee` and `vbytes`.
+    pub fn from_vb(fee: u64, vbytes: usize) -> FeeRate {
+        let rate = fee as f32 / vbytes as f32;
+        Self::from_sat_per_vb(rate)
+    }
+
     /// Return the value as satoshi/vbyte
     pub fn as_sat_vb(&self) -> f32 {
         self.0
     }
+
+    /// Calculate absolute fee in Satoshis using size in weight units.
+    pub fn fee_wu(&self, wu: usize) -> u64 {
+        self.fee_vb(wu.vbytes())
+    }
+
+    /// Calculate absolute fee in Satoshis using size in virtual bytes.
+    pub fn fee_vb(&self, vbytes: usize) -> u64 {
+        (self.as_sat_vb() * vbytes as f32).ceil() as u64
+    }
 }
 
 impl std::default::Default for FeeRate {
@@ -77,10 +99,31 @@ impl std::default::Default for FeeRate {
     }
 }
 
+impl Sub for FeeRate {
+    type Output = Self;
+
+    fn sub(self, other: FeeRate) -> Self::Output {
+        FeeRate(self.0 - other.0)
+    }
+}
+
+/// Trait implemented by types that can be used to measure weight units.
+pub trait Vbytes {
+    /// Convert weight units to virtual bytes.
+    fn vbytes(self) -> usize;
+}
+
+impl Vbytes for usize {
+    fn vbytes(self) -> usize {
+        // ref: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki#transaction-size-calculations
+        (self as f32 / 4.0).ceil() as usize
+    }
+}
+
 /// An unspent output owned by a [`Wallet`].
 ///
 /// [`Wallet`]: crate::Wallet
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
 pub struct LocalUtxo {
     /// Reference to a transaction output
     pub outpoint: OutPoint,
@@ -139,7 +182,7 @@ impl Utxo {
                 }
 
                 if let Some(txout) = &psbt_input.witness_utxo {
-                    return &txout;
+                    return txout;
                 }
 
                 unreachable!("Foreign UTXOs will always have one of these set")
@@ -155,16 +198,47 @@ pub struct TransactionDetails {
     pub transaction: Option<Transaction>,
     /// Transaction id
     pub txid: Txid,
-    /// Timestamp
-    pub timestamp: u64,
+
     /// Received value (sats)
     pub received: u64,
     /// Sent value (sats)
     pub sent: u64,
-    /// Fee value (sats)
-    pub fees: u64,
-    /// Confirmed in block height, `None` means unconfirmed
-    pub height: Option<u32>,
+    /// Fee value (sats) if available.
+    /// The availability of the fee depends on the backend. It's never `None` with an Electrum
+    /// Server backend, but it could be `None` with a Bitcoin RPC node without txindex that receive
+    /// funds while offline.
+    pub fee: Option<u64>,
+    /// If the transaction is confirmed, contains height and timestamp of the block containing the
+    /// transaction, unconfirmed transaction contains `None`.
+    pub confirmation_time: Option<ConfirmationTime>,
+    /// Whether the tx has been verified against the consensus rules
+    ///
+    /// Confirmed txs are considered "verified" by default, while unconfirmed txs are checked to
+    /// ensure an unstrusted [`Blockchain`](crate::blockchain::Blockchain) backend can't trick the
+    /// wallet into using an invalid tx as an RBF template.
+    ///
+    /// The check is only perfomed when the `verify` feature is enabled.
+    #[serde(default = "bool::default")] // default to `false` if not specified
+    pub verified: bool,
+}
+
+/// Block height and timestamp of the block containing the confirmed transaction
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
+pub struct ConfirmationTime {
+    /// confirmation block height
+    pub height: u32,
+    /// confirmation block timestamp
+    pub timestamp: u64,
+}
+
+impl ConfirmationTime {
+    /// Returns `Some` `ConfirmationTime` if both `height` and `timestamp` are `Some`
+    pub fn new(height: Option<u32>, timestamp: Option<u64>) -> Option<Self> {
+        match (height, timestamp) {
+            (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+            _ => None,
+        }
+    }
 }
 
 #[cfg(test)]

src/wallet/address_validator.rs

@@ -115,8 +115,8 @@ mod test {
     use std::sync::Arc;
 
     use super::*;
-    use crate::wallet::test::{get_funded_wallet, get_test_wpkh};
     use crate::wallet::AddressIndex::New;
+    use crate::wallet::{get_funded_wallet, test::get_test_wpkh};
 
     #[derive(Debug)]
     struct TestValidator;

src/wallet/coin_selection.rs

@@ -26,7 +26,7 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::wallet::coin_selection::*;
+//! # use bdk::wallet::{self, coin_selection::*};
 //! # use bdk::database::Database;
 //! # use bdk::*;
 //! # const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4 + 1) * 4;
@@ -41,7 +41,7 @@
 //!         optional_utxos: Vec<WeightedUtxo>,
 //!         fee_rate: FeeRate,
 //!         amount_needed: u64,
-//!         fee_amount: f32,
+//!         fee_amount: u64,
 //!     ) -> Result<CoinSelectionResult, bdk::Error> {
 //!         let mut selected_amount = 0;
 //!         let mut additional_weight = 0;
@@ -57,9 +57,8 @@
 //!                 },
 //!             )
 //!             .collect::<Vec<_>>();
-//!         let additional_fees = additional_weight as f32 * fee_rate.as_sat_vb() / 4.0;
-//!         let amount_needed_with_fees =
-//!             (fee_amount + additional_fees).ceil() as u64 + amount_needed;
+//!         let additional_fees = fee_rate.fee_wu(additional_weight);
+//!         let amount_needed_with_fees = (fee_amount + additional_fees) + amount_needed;
 //!         if amount_needed_with_fees > selected_amount {
 //!             return Err(bdk::Error::InsufficientFunds {
 //!                 needed: amount_needed_with_fees,
@@ -117,7 +116,7 @@ pub struct CoinSelectionResult {
     /// List of outputs selected for use as inputs
     pub selected: Vec<Utxo>,
     /// Total fee amount in satoshi
-    pub fee_amount: f32,
+    pub fee_amount: u64,
 }
 
 impl CoinSelectionResult {
@@ -164,7 +163,7 @@ pub trait CoinSelectionAlgorithm<D: Database>: std::fmt::Debug {
         optional_utxos: Vec<WeightedUtxo>,
         fee_rate: FeeRate,
         amount_needed: u64,
-        fee_amount: f32,
+        fee_amount: u64,
     ) -> Result<CoinSelectionResult, Error>;
 }
 
@@ -183,10 +182,8 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
         mut optional_utxos: Vec<WeightedUtxo>,
         fee_rate: FeeRate,
         amount_needed: u64,
-        mut fee_amount: f32,
+        mut fee_amount: u64,
     ) -> Result<CoinSelectionResult, Error> {
-        let calc_fee_bytes = |wu| (wu as f32) * fee_rate.as_sat_vb() / 4.0;
-
         log::debug!(
             "amount_needed = `{}`, fee_amount = `{}`, fee_rate = `{:?}`",
             amount_needed,
@@ -211,9 +208,9 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
             .scan(
                 (&mut selected_amount, &mut fee_amount),
                 |(selected_amount, fee_amount), (must_use, weighted_utxo)| {
-                    if must_use || **selected_amount < amount_needed + (fee_amount.ceil() as u64) {
+                    if must_use || **selected_amount < amount_needed + **fee_amount {
                         **fee_amount +=
-                            calc_fee_bytes(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
+                            fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
                         **selected_amount += weighted_utxo.utxo.txout().value;
 
                         log::debug!(
@@ -230,7 +227,7 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
             )
             .collect::<Vec<_>>();
 
-        let amount_needed_with_fees = amount_needed + (fee_amount.ceil() as u64);
+        let amount_needed_with_fees = amount_needed + fee_amount;
         if selected_amount < amount_needed_with_fees {
             return Err(Error::InsufficientFunds {
                 needed: amount_needed_with_fees,
@@ -250,16 +247,15 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
 struct OutputGroup {
     weighted_utxo: WeightedUtxo,
     // Amount of fees for spending a certain utxo, calculated using a certain FeeRate
-    fee: f32,
+    fee: u64,
     // The effective value of the UTXO, i.e., the utxo value minus the fee for spending it
     effective_value: i64,
 }
 
 impl OutputGroup {
     fn new(weighted_utxo: WeightedUtxo, fee_rate: FeeRate) -> Self {
-        let fee = (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as f32 / 4.0
-            * fee_rate.as_sat_vb();
-        let effective_value = weighted_utxo.utxo.txout().value as i64 - fee.ceil() as i64;
+        let fee = fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
+        let effective_value = weighted_utxo.utxo.txout().value as i64 - fee as i64;
         OutputGroup {
             weighted_utxo,
             fee,
@@ -302,7 +298,7 @@ impl<D: Database> CoinSelectionAlgorithm<D> for BranchAndBoundCoinSelection {
         optional_utxos: Vec<WeightedUtxo>,
         fee_rate: FeeRate,
         amount_needed: u64,
-        fee_amount: f32,
+        fee_amount: u64,
     ) -> Result<CoinSelectionResult, Error> {
         // Mapping every (UTXO, usize) to an output group
         let required_utxos: Vec<OutputGroup> = required_utxos
@@ -324,7 +320,7 @@ impl<D: Database> CoinSelectionAlgorithm<D> for BranchAndBoundCoinSelection {
             .iter()
             .fold(0, |acc, x| acc + x.effective_value);
 
-        let actual_target = fee_amount.ceil() as u64 + amount_needed;
+        let actual_target = fee_amount + amount_needed;
         let cost_of_change = self.size_of_change as f32 * fee_rate.as_sat_vb();
 
         let expected = (curr_available_value + curr_value)
@@ -344,6 +340,14 @@ impl<D: Database> CoinSelectionAlgorithm<D> for BranchAndBoundCoinSelection {
             .try_into()
             .expect("Bitcoin amount to fit into i64");
 
+        if curr_value > actual_target {
+            return Ok(BranchAndBoundCoinSelection::calculate_cs_result(
+                vec![],
+                required_utxos,
+                fee_amount,
+            ));
+        }
+
         Ok(self
             .bnb(
                 required_utxos.clone(),
@@ -377,7 +381,7 @@ impl BranchAndBoundCoinSelection {
         mut curr_value: i64,
         mut curr_available_value: i64,
         actual_target: i64,
-        fee_amount: f32,
+        fee_amount: u64,
         cost_of_change: f32,
     ) -> Result<CoinSelectionResult, Error> {
         // current_selection[i] will contain true if we are using optional_utxos[i],
@@ -485,7 +489,7 @@ impl BranchAndBoundCoinSelection {
         mut optional_utxos: Vec<OutputGroup>,
         curr_value: i64,
         actual_target: i64,
-        fee_amount: f32,
+        fee_amount: u64,
     ) -> CoinSelectionResult {
         #[cfg(not(test))]
         optional_utxos.shuffle(&mut thread_rng());
@@ -514,10 +518,10 @@ impl BranchAndBoundCoinSelection {
     fn calculate_cs_result(
         mut selected_utxos: Vec<OutputGroup>,
         mut required_utxos: Vec<OutputGroup>,
-        mut fee_amount: f32,
+        mut fee_amount: u64,
     ) -> CoinSelectionResult {
         selected_utxos.append(&mut required_utxos);
-        fee_amount += selected_utxos.iter().map(|u| u.fee).sum::<f32>();
+        fee_amount += selected_utxos.iter().map(|u| u.fee).sum::<u64>();
         let selected = selected_utxos
             .into_iter()
             .map(|u| u.weighted_utxo.utxo)
@@ -539,6 +543,7 @@ mod test {
     use super::*;
     use crate::database::MemoryDatabase;
     use crate::types::*;
+    use crate::wallet::Vbytes;
 
     use rand::rngs::StdRng;
     use rand::seq::SliceRandom;
@@ -546,52 +551,33 @@ mod test {
 
     const P2WPKH_WITNESS_SIZE: usize = 73 + 33 + 2;
 
-    const FEE_AMOUNT: f32 = 50.0;
+    const FEE_AMOUNT: u64 = 50;
+
+    fn utxo(value: u64, index: u32) -> WeightedUtxo {
+        assert!(index < 10);
+        let outpoint = OutPoint::from_str(&format!(
+            "000000000000000000000000000000000000000000000000000000000000000{}:0",
+            index
+        ))
+        .unwrap();
+        WeightedUtxo {
+            satisfaction_weight: P2WPKH_WITNESS_SIZE,
+            utxo: Utxo::Local(LocalUtxo {
+                outpoint,
+                txout: TxOut {
+                    value,
+                    script_pubkey: Script::new(),
+                },
+                keychain: KeychainKind::External,
+            }),
+        }
+    }
 
     fn get_test_utxos() -> Vec<WeightedUtxo> {
         vec![
-            WeightedUtxo {
-                satisfaction_weight: P2WPKH_WITNESS_SIZE,
-                utxo: Utxo::Local(LocalUtxo {
-                    outpoint: OutPoint::from_str(
-                        "0000000000000000000000000000000000000000000000000000000000000000:0",
-                    )
-                    .unwrap(),
-                    txout: TxOut {
-                        value: 100_000,
-                        script_pubkey: Script::new(),
-                    },
-                    keychain: KeychainKind::External,
-                }),
-            },
-            WeightedUtxo {
-                satisfaction_weight: P2WPKH_WITNESS_SIZE,
-                utxo: Utxo::Local(LocalUtxo {
-                    outpoint: OutPoint::from_str(
-                        "0000000000000000000000000000000000000000000000000000000000000001:0",
-                    )
-                    .unwrap(),
-                    txout: TxOut {
-                        value: FEE_AMOUNT as u64 - 40,
-                        script_pubkey: Script::new(),
-                    },
-                    keychain: KeychainKind::External,
-                }),
-            },
-            WeightedUtxo {
-                satisfaction_weight: P2WPKH_WITNESS_SIZE,
-                utxo: Utxo::Local(LocalUtxo {
-                    outpoint: OutPoint::from_str(
-                        "0000000000000000000000000000000000000000000000000000000000000002:0",
-                    )
-                    .unwrap(),
-                    txout: TxOut {
-                        value: 200_000,
-                        script_pubkey: Script::new(),
-                    },
-                    keychain: KeychainKind::Internal,
-                }),
-            },
+            utxo(100_000, 0),
+            utxo(FEE_AMOUNT as u64 - 40, 1),
+            utxo(200_000, 2),
         ]
     }
 
@@ -655,13 +641,13 @@ mod test {
                 vec![],
                 FeeRate::from_sat_per_vb(1.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 3);
         assert_eq!(result.selected_amount(), 300_010);
-        assert!((result.fee_amount - 254.0).abs() < f32::EPSILON);
+        assert_eq!(result.fee_amount, 254)
     }
 
     #[test]
@@ -676,13 +662,13 @@ mod test {
                 vec![],
                 FeeRate::from_sat_per_vb(1.0),
                 20_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 3);
         assert_eq!(result.selected_amount(), 300_010);
-        assert!((result.fee_amount - 254.0).abs() < f32::EPSILON);
+        assert_eq!(result.fee_amount, 254);
     }
 
     #[test]
@@ -697,13 +683,13 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 20_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 1);
         assert_eq!(result.selected_amount(), 200_000);
-        assert!((result.fee_amount - 118.0).abs() < f32::EPSILON);
+        assert_eq!(result.fee_amount, 118);
     }
 
     #[test]
@@ -719,7 +705,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 500_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -737,7 +723,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1000.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -757,13 +743,13 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 3);
         assert_eq!(result.selected_amount(), 300_000);
-        assert!((result.fee_amount - 254.0).abs() < f32::EPSILON);
+        assert_eq!(result.fee_amount, 254);
     }
 
     #[test]
@@ -784,7 +770,7 @@ mod test {
 
         assert_eq!(result.selected.len(), 3);
         assert_eq!(result.selected_amount(), 300_010);
-        assert!((result.fee_amount - 254.0).abs() < f32::EPSILON);
+        assert_eq!(result.fee_amount, 254);
     }
 
     #[test]
@@ -805,7 +791,38 @@ mod test {
 
         assert_eq!(result.selected.len(), 3);
         assert_eq!(result.selected_amount(), 300010);
-        assert!((result.fee_amount - 254.0).abs() < f32::EPSILON);
+        assert_eq!(result.fee_amount, 254);
+    }
+
+    #[test]
+    fn test_bnb_coin_selection_required_not_enough() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let required = vec![utxos[0].clone()];
+        let mut optional = utxos[1..].to_vec();
+        optional.push(utxo(500_000, 3));
+
+        // Defensive assertions, for sanity and in case someone changes the test utxos vector.
+        let amount: u64 = required.iter().map(|u| u.utxo.txout().value).sum();
+        assert_eq!(amount, 100_000);
+        let amount: u64 = optional.iter().map(|u| u.utxo.txout().value).sum();
+        assert!(amount > 150_000);
+
+        let result = BranchAndBoundCoinSelection::default()
+            .coin_select(
+                &database,
+                required,
+                optional,
+                FeeRate::from_sat_per_vb(1.0),
+                150_000,
+                FEE_AMOUNT,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 3);
+        assert_eq!(result.selected_amount(), 300_010);
+        assert!((result.fee_amount as f32 - 254.0).abs() < f32::EPSILON);
     }
 
     #[test]
@@ -821,7 +838,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 500_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -839,7 +856,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1000.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -856,15 +873,15 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 99932, // first utxo's effective value
-                0.0,
+                0,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 1);
         assert_eq!(result.selected_amount(), 100_000);
-        let input_size = (TXIN_BASE_WEIGHT as f32) / 4.0 + P2WPKH_WITNESS_SIZE as f32 / 4.0;
+        let input_size = (TXIN_BASE_WEIGHT + P2WPKH_WITNESS_SIZE).vbytes();
         let epsilon = 0.5;
-        assert!((1.0 - (result.fee_amount / input_size)).abs() < epsilon);
+        assert!((1.0 - (result.fee_amount as f32 / input_size as f32)).abs() < epsilon);
     }
 
     #[test]
@@ -883,7 +900,7 @@ mod test {
                     optional_utxos,
                     FeeRate::from_sat_per_vb(0.0),
                     target_amount,
-                    0.0,
+                    0,
                 )
                 .unwrap();
             assert_eq!(result.selected_amount(), target_amount);
@@ -910,7 +927,7 @@ mod test {
                 0,
                 curr_available_value,
                 20_000,
-                50.0,
+                FEE_AMOUNT,
                 cost_of_change,
             )
             .unwrap();
@@ -937,7 +954,7 @@ mod test {
                 0,
                 curr_available_value,
                 20_000,
-                50.0,
+                FEE_AMOUNT,
                 cost_of_change,
             )
             .unwrap();
@@ -949,7 +966,6 @@ mod test {
         let fee_rate = FeeRate::from_sat_per_vb(1.0);
         let size_of_change = 31;
         let cost_of_change = size_of_change as f32 * fee_rate.as_sat_vb();
-        let fee_amount = 50.0;
 
         let utxos: Vec<_> = generate_same_value_utxos(50_000, 10)
             .into_iter()
@@ -971,12 +987,12 @@ mod test {
                 curr_value,
                 curr_available_value,
                 target_amount,
-                fee_amount,
+                FEE_AMOUNT,
                 cost_of_change,
             )
             .unwrap();
-        assert!((result.fee_amount - 186.0).abs() < f32::EPSILON);
         assert_eq!(result.selected_amount(), 100_000);
+        assert_eq!(result.fee_amount, 186);
     }
 
     // TODO: bnb() function should be optimized, and this test should be done with more utxos
@@ -1008,7 +1024,7 @@ mod test {
                     curr_value,
                     curr_available_value,
                     target_amount,
-                    0.0,
+                    0,
                     0.0,
                 )
                 .unwrap();
@@ -1034,12 +1050,10 @@ mod test {
             utxos,
             0,
             target_amount as i64,
-            50.0,
+            FEE_AMOUNT,
         );
 
         assert!(result.selected_amount() > target_amount);
-        assert!(
-            (result.fee_amount - (50.0 + result.selected.len() as f32 * 68.0)).abs() < f32::EPSILON
-        );
+        assert_eq!(result.fee_amount, (50 + result.selected.len() * 68) as u64);
     }
 }

src/wallet/export.rs

@@ -128,7 +128,7 @@ impl WalletExport {
             Ok(txs) => {
                 let mut heights = txs
                     .into_iter()
-                    .map(|tx| tx.height.unwrap_or(0))
+                    .map(|tx| tx.confirmation_time.map(|c| c.height).unwrap_or(0))
                     .collect::<Vec<_>>();
                 heights.sort_unstable();
 
@@ -212,6 +212,7 @@ mod test {
     use crate::database::{memory::MemoryDatabase, BatchOperations};
     use crate::types::TransactionDetails;
     use crate::wallet::Wallet;
+    use crate::ConfirmationTime;
 
     fn get_test_db() -> MemoryDatabase {
         let mut db = MemoryDatabase::new();
@@ -221,11 +222,15 @@ mod test {
                 "4ddff1fa33af17f377f62b72357b43107c19110a8009b36fb832af505efed98a",
             )
             .unwrap(),
-            timestamp: 12345678,
+
             received: 100_000,
             sent: 0,
-            fees: 500,
-            height: Some(5000),
+            fee: Some(500),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 12345678,
+                height: 5000,
+            }),
+            verified: true,
         })
         .unwrap();
 

src/wallet/signer.rs

@@ -222,7 +222,7 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
             .bip32_derivation
             .iter()
             .filter_map(|(pk, &(fingerprint, ref path))| {
-                if self.matches(&(fingerprint, path.clone()), &secp).is_some() {
+                if self.matches(&(fingerprint, path.clone()), secp).is_some() {
                     Some((pk, path))
                 } else {
                     None
@@ -240,12 +240,12 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
                     &full_path.into_iter().cloned().collect::<Vec<ChildNumber>>()
                         [origin_path.len()..],
                 );
-                self.xkey.derive_priv(&secp, &deriv_path).unwrap()
+                self.xkey.derive_priv(secp, &deriv_path).unwrap()
             }
-            None => self.xkey.derive_priv(&secp, &full_path).unwrap(),
+            None => self.xkey.derive_priv(secp, &full_path).unwrap(),
         };
 
-        if &derived_key.private_key.public_key(&secp) != public_key {
+        if &derived_key.private_key.public_key(secp) != public_key {
             Err(SignerError::InvalidKey)
         } else {
             derived_key.private_key.sign(psbt, Some(input_index), secp)
@@ -257,7 +257,7 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
     }
 
     fn id(&self, secp: &SecpCtx) -> SignerId {
-        SignerId::from(self.root_fingerprint(&secp))
+        SignerId::from(self.root_fingerprint(secp))
     }
 
     fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
@@ -283,7 +283,7 @@ impl Signer for PrivateKey {
             return Ok(());
         }
 
-        let pubkey = self.public_key(&secp);
+        let pubkey = self.public_key(secp);
         if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
             return Ok(());
         }
@@ -591,7 +591,7 @@ impl ComputeSighash for Segwitv0 {
                     .map(Script::is_v0_p2wpkh)
                     .unwrap_or(false)
                 {
-                    p2wpkh_script_code(&psbt_input.redeem_script.as_ref().unwrap())
+                    p2wpkh_script_code(psbt_input.redeem_script.as_ref().unwrap())
                 } else {
                     return Err(SignerError::MissingWitnessScript);
                 }

src/wallet/tx_builder.rs

@@ -133,7 +133,7 @@ pub struct TxBuilder<'a, B, D, Cs, Ctx> {
 pub(crate) struct TxParams {
     pub(crate) recipients: Vec<(Script, u64)>,
     pub(crate) drain_wallet: bool,
-    pub(crate) single_recipient: Option<Script>,
+    pub(crate) drain_to: Option<Script>,
     pub(crate) fee_policy: Option<FeePolicy>,
     pub(crate) internal_policy_path: Option<BTreeMap<String, Vec<usize>>>,
     pub(crate) external_policy_path: Option<BTreeMap<String, Vec<usize>>>,
@@ -560,49 +560,81 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>> TxBuilder<'a, B, D,
         self
     }
 
-    /// Set a single recipient that will get all the selected funds minus the fee. No change will
-    /// be created
+    /// Sets the address to *drain* excess coins to.
     ///
-    /// This method overrides any recipient set with [`set_recipients`](Self::set_recipients) or
-    /// [`add_recipient`](Self::add_recipient).
+    /// Usually, when there are excess coins they are sent to a change address generated by the
+    /// wallet. This option replaces the usual change address with an arbitrary `script_pubkey` of
+    /// your choosing. Just as with a change output, if the drain output is not needed (the excess
+    /// coins are too small) it will not be included in the resulting transaction. The only
+    /// difference is that it is valid to use `drain_to` without setting any ordinary recipients
+    /// with [`add_recipient`] (but it is perfectly fine to add recipients as well).
     ///
-    /// It can only be used in conjunction with [`drain_wallet`](Self::drain_wallet) to send the
-    /// entire content of the wallet (minus filters) to a single recipient or with a
-    /// list of manually selected UTXOs by enabling [`manually_selected_only`](Self::manually_selected_only)
-    /// and selecting them with or [`add_utxo`](Self::add_utxo).
+    /// When bumping the fees of a transaction made with this option, you probably want to
+    /// use [`allow_shrinking`] to allow this output to be reduced to pay for the extra fees.
     ///
-    /// When bumping the fees of a transaction made with this option, the user should remeber to
-    /// add [`maintain_single_recipient`](Self::maintain_single_recipient) to correctly update the
-    /// single output instead of adding one more for the change.
-    pub fn set_single_recipient(&mut self, recipient: Script) -> &mut Self {
-        self.params.single_recipient = Some(recipient);
-        self.params.recipients.clear();
-
+    /// # Example
+    ///
+    /// `drain_to` is very useful for draining all the coins in a wallet with [`drain_wallet`] to a
+    /// single address.
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// # use bitcoin::*;
+    /// # use bdk::*;
+    /// # use bdk::wallet::tx_builder::CreateTx;
+    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// # let wallet = doctest_wallet!();
+    /// let mut tx_builder = wallet.build_tx();
+    ///
+    /// tx_builder
+    ///     // Spend all outputs in this wallet.
+    ///     .drain_wallet()
+    ///     // Send the excess (which is all the coins minus the fee) to this address.
+    ///     .drain_to(to_address.script_pubkey())
+    ///     .fee_rate(FeeRate::from_sat_per_vb(5.0))
+    ///     .enable_rbf();
+    /// let (psbt, tx_details) = tx_builder.finish()?;
+    /// # Ok::<(), bdk::Error>(())
+    /// ```
+    ///
+    /// [`allow_shrinking`]: Self::allow_shrinking
+    /// [`add_recipient`]: Self::add_recipient
+    /// [`drain_wallet`]: Self::drain_wallet
+    pub fn drain_to(&mut self, script_pubkey: Script) -> &mut Self {
+        self.params.drain_to = Some(script_pubkey);
         self
     }
 }
 
 // methods supported only by bump_fee
 impl<'a, B, D: BatchDatabase> TxBuilder<'a, B, D, DefaultCoinSelectionAlgorithm, BumpFee> {
-    /// Bump the fees of a transaction made with [`set_single_recipient`](Self::set_single_recipient)
+    /// Explicitly tells the wallet that it is allowed to reduce the fee of the output matching this
+    /// `script_pubkey` in order to bump the transaction fee. Without specifying this the wallet
+    /// will attempt to find a change output to shrink instead.
     ///
-    /// Unless extra inputs are specified with [`add_utxo`], this flag will make
-    /// `bump_fee` reduce the value of the existing output, or fail if it would be consumed
-    /// entirely given the higher new fee rate.
+    /// **Note** that the output may shrink to below the dust limit and therefore be removed. If it is
+    /// preserved then it is currently not guaranteed to be in the same position as it was
+    /// originally.
     ///
-    /// If extra inputs are added and they are not entirely consumed in fees, a change output will not
-    /// be added; the existing output will simply grow in value.
-    ///
-    /// Fails if the transaction has more than one outputs.
-    ///
-    /// [`add_utxo`]: Self::add_utxo
-    pub fn maintain_single_recipient(&mut self) -> Result<&mut Self, Error> {
-        let mut recipients = self.params.recipients.drain(..).collect::<Vec<_>>();
-        if recipients.len() != 1 {
-            return Err(Error::SingleRecipientMultipleOutputs);
+    /// Returns an `Err` if `script_pubkey` can't be found among the recipients of the
+    /// transaction we are bumping.
+    pub fn allow_shrinking(&mut self, script_pubkey: Script) -> Result<&mut Self, Error> {
+        match self
+            .params
+            .recipients
+            .iter()
+            .position(|(recipient_script, _)| *recipient_script == script_pubkey)
+        {
+            Some(position) => {
+                self.params.recipients.remove(position);
+                self.params.drain_to = Some(script_pubkey);
+                Ok(self)
+            }
+            None => Err(Error::Generic(format!(
+                "{} was not in the original transaction",
+                script_pubkey
+            ))),
         }
-        self.params.single_recipient = Some(recipients.pop().unwrap().0);
-        Ok(self)
     }
 }
 

src/wallet/verify.rs

@@ -0,0 +1,185 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Verify transactions against the consensus rules
+
+use std::collections::HashMap;
+use std::fmt;
+
+use bitcoin::consensus::serialize;
+use bitcoin::{OutPoint, Transaction, Txid};
+
+use crate::blockchain::Blockchain;
+use crate::database::Database;
+use crate::error::Error;
+
+/// Verify a transaction against the consensus rules
+///
+/// This function uses [`bitcoinconsensus`] to verify transactions by fetching the required data
+/// either from the [`Database`] or using the [`Blockchain`].
+///
+/// Depending on the [capabilities](crate::blockchain::Blockchain::get_capabilities) of the
+/// [`Blockchain`] backend, the method could fail when called with old "historical" transactions or
+/// with unconfirmed transactions that have been evicted from the backend's memory.
+pub fn verify_tx<D: Database, B: Blockchain>(
+    tx: &Transaction,
+    database: &D,
+    blockchain: &B,
+) -> Result<(), VerifyError> {
+    log::debug!("Verifying {}", tx.txid());
+
+    let serialized_tx = serialize(tx);
+    let mut tx_cache = HashMap::<_, Transaction>::new();
+
+    for (index, input) in tx.input.iter().enumerate() {
+        let prev_tx = if let Some(prev_tx) = tx_cache.get(&input.previous_output.txid) {
+            prev_tx.clone()
+        } else if let Some(prev_tx) = database.get_raw_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else if let Some(prev_tx) = blockchain.get_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else {
+            return Err(VerifyError::MissingInputTx(input.previous_output.txid));
+        };
+
+        let spent_output = prev_tx
+            .output
+            .get(input.previous_output.vout as usize)
+            .ok_or(VerifyError::InvalidInput(input.previous_output))?;
+
+        bitcoinconsensus::verify(
+            &spent_output.script_pubkey.to_bytes(),
+            spent_output.value,
+            &serialized_tx,
+            index,
+        )?;
+
+        // Since we have a local cache we might as well cache stuff from the db, as it will very
+        // likely decrease latency compared to reading from disk or performing an SQL query.
+        tx_cache.insert(prev_tx.txid(), prev_tx);
+    }
+
+    Ok(())
+}
+
+/// Error during validation of a tx agains the consensus rules
+#[derive(Debug)]
+pub enum VerifyError {
+    /// The transaction being spent is not available in the database or the blockchain client
+    MissingInputTx(Txid),
+    /// The transaction being spent doesn't have the requested output
+    InvalidInput(OutPoint),
+
+    /// Consensus error
+    Consensus(bitcoinconsensus::Error),
+
+    /// Generic error
+    ///
+    /// It has to be wrapped in a `Box` since `Error` has a variant that contains this enum
+    Global(Box<Error>),
+}
+
+impl fmt::Display for VerifyError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for VerifyError {}
+
+impl From<Error> for VerifyError {
+    fn from(other: Error) -> Self {
+        VerifyError::Global(Box::new(other))
+    }
+}
+impl_error!(bitcoinconsensus::Error, Consensus, VerifyError);
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashSet;
+
+    use bitcoin::consensus::encode::deserialize;
+    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::{Transaction, Txid};
+
+    use crate::blockchain::{Blockchain, Capability, Progress};
+    use crate::database::{BatchDatabase, BatchOperations, MemoryDatabase};
+    use crate::FeeRate;
+
+    use super::*;
+
+    struct DummyBlockchain;
+
+    impl Blockchain for DummyBlockchain {
+        fn get_capabilities(&self) -> HashSet<Capability> {
+            Default::default()
+        }
+        fn setup<D: BatchDatabase, P: 'static + Progress>(
+            &self,
+            _database: &mut D,
+            _progress_update: P,
+        ) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_tx(&self, _txid: &Txid) -> Result<Option<Transaction>, Error> {
+            Ok(None)
+        }
+        fn broadcast(&self, _tx: &Transaction) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_height(&self) -> Result<u32, Error> {
+            Ok(42)
+        }
+        fn estimate_fee(&self, _target: usize) -> Result<FeeRate, Error> {
+            Ok(FeeRate::default_min_relay_fee())
+        }
+    }
+
+    #[test]
+    fn test_verify_fail_unsigned_tx() {
+        // https://blockstream.info/tx/95da344585fcf2e5f7d6cbf2c3df2dcce84f9196f7a7bb901a43275cd6eb7c3f
+        let prev_tx: Transaction = deserialize(&Vec::<u8>::from_hex("020000000101192dea5e66d444380e106f8e53acb171703f00d43fb6b3ae88ca5644bdb7e1000000006b48304502210098328d026ce138411f957966c1cf7f7597ccbb170f5d5655ee3e9f47b18f6999022017c3526fc9147830e1340e04934476a3d1521af5b4de4e98baf49ec4c072079e01210276f847f77ec8dd66d78affd3c318a0ed26d89dab33fa143333c207402fcec352feffffff023d0ac203000000001976a9144bfbaf6afb76cc5771bc6404810d1cc041a6933988aca4b956050000000017a91494d5543c74a3ee98e0cf8e8caef5dc813a0f34b48768cb0700").unwrap()).unwrap();
+        // https://blockstream.info/tx/aca326a724eda9a461c10a876534ecd5ae7b27f10f26c3862fb996f80ea2d45d
+        let signed_tx: Transaction = deserialize(&Vec::<u8>::from_hex("02000000013f7cebd65c27431a90bba7f796914fe8cc2ddfc3f2cbd6f7e5f2fc854534da95000000006b483045022100de1ac3bcdfb0332207c4a91f3832bd2c2915840165f876ab47c5f8996b971c3602201c6c053d750fadde599e6f5c4e1963df0f01fc0d97815e8157e3d59fe09ca30d012103699b464d1d8bc9e47d4fb1cdaa89a1c5783d68363c4dbc4b524ed3d857148617feffffff02836d3c01000000001976a914fc25d6d5c94003bf5b0c7b640a248e2c637fcfb088ac7ada8202000000001976a914fbed3d9b11183209a57999d54d59f67c019e756c88ac6acb0700").unwrap()).unwrap();
+
+        let mut database = MemoryDatabase::new();
+        let blockchain = DummyBlockchain;
+
+        let mut unsigned_tx = signed_tx.clone();
+        for input in &mut unsigned_tx.input {
+            input.script_sig = Default::default();
+            input.witness = Default::default();
+        }
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail with missing input tx");
+        assert!(
+            matches!(result, Err(VerifyError::MissingInputTx(txid)) if txid == prev_tx.txid()),
+            "Error should be a `MissingInputTx` error"
+        );
+
+        // insert the prev_tx
+        database.set_raw_tx(&prev_tx).unwrap();
+
+        let result = verify_tx(&unsigned_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail since the TX is unsigned");
+        assert!(
+            matches!(result, Err(VerifyError::Consensus(_))),
+            "Error should be a `Consensus` error"
+        );
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(
+            result.is_ok(),
+            "Should work since the TX is correctly signed"
+        );
+    }
+}