Bump version to 0.13.0

(cherry picked from commit d75d221540)

.github/workflows/code_coverage.yml

@@ -3,25 +3,35 @@ on: [push]
 name: Code Coverage
 
 jobs:
-  tarpaulin-codecov:
-    name: Tarpaulin to codecov.io
+
+  Codecov:
+    name: Code Coverage
     runs-on: ubuntu-latest
+    env:
+      CARGO_INCREMENTAL: '0'
+      RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off'
+      RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off'
+
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
+      - name: Install rustup
+        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
       - name: Set default toolchain
         run: rustup default nightly
       - name: Set profile
         run: rustup set profile minimal
+      - name: Update toolchain
+        run: rustup update
+      - name: Test
+        run: cargo test --features all-keys,compiler,esplora,ureq,compact_filters --no-default-features
+                    
+      - id: coverage
+        name: Generate coverage
+        uses: actions-rs/grcov@v0.1.5
 
-      - name: Install tarpaulin
-        run: cargo install cargo-tarpaulin
-      - name: Tarpaulin
-        run: cargo tarpaulin --features all-keys,compiler,esplora,compact_filters --run-types Tests,Doctests --exclude-files "testutils/*" --out Xml
-
-      - name: Publish to codecov.io
-        uses: codecov/codecov-action@v1.0.15
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
         with:
-          fail_ci_if_error: true
-          file: ./cobertura.xml
+          file: ${{ steps.coverage.outputs.report }}
+          directory: ./coverage/reports/

.github/workflows/cont_integration.yml

@@ -10,18 +10,23 @@ jobs:
     strategy:
       matrix:
         rust:
-          - stable
-          - 1.45.0 # MSRV
+          - 1.56.0 # STABLE
+          - 1.46.0 # MSRV
         features:
           - default
           - minimal
           - all-keys
-          - minimal,esplora
+          - minimal,use-esplora-ureq
           - key-value-db
           - electrum
           - compact_filters
-          - esplora,key-value-db,electrum
+          - esplora,ureq,key-value-db,electrum
           - compiler
+          - rpc
+          - verify
+          - async-interface
+          - use-esplora-reqwest
+          - sqlite
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -46,7 +51,7 @@ jobs:
       - name: Build
         run: cargo build --features ${{ matrix.features }} --no-default-features
       - name: Clippy
-        run: cargo clippy --features ${{ matrix.features }} --no-default-features -- -D warnings
+        run: cargo clippy --all-targets --features ${{ matrix.features }} --no-default-features -- -D warnings
       - name: Test
         run: cargo test --features ${{ matrix.features }} --no-default-features
 
@@ -73,16 +78,21 @@ jobs:
       - name: Test
         run: cargo test --features test-md-docs --no-default-features -- doctest::ReadmeDoctests
 
-  test-electrum:
-    name: Test electrum
-    runs-on: ubuntu-16.04
-    container: bitcoindevkit/electrs
-    env:
-      MAGICAL_RPC_AUTH: USER_PASS
-      MAGICAL_RPC_USER: admin
-      MAGICAL_RPC_PASS: passw
-      MAGICAL_RPC_URL: 127.0.0.1:18443
-      MAGICAL_ELECTRUM_URL: tcp://127.0.0.1:60401
+  test-blockchains:
+    name: Blockchain ${{ matrix.blockchain.features }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        blockchain:
+          - name: electrum
+            features: test-electrum
+          - name: rpc
+            features: test-rpc
+          - name: esplora
+            features: test-esplora,use-esplora-reqwest
+          - name: esplora
+            features: test-esplora,use-esplora-ureq    
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -94,22 +104,17 @@ jobs:
             ~/.cargo/git
             target
           key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
-      - name: Install rustup
-        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - name: Set default toolchain
-        run: $HOME/.cargo/bin/rustup default stable
-      - name: Set profile
-        run: $HOME/.cargo/bin/rustup set profile minimal
-      - name: Update toolchain
-        run: $HOME/.cargo/bin/rustup update
-      - name: Start core
-        run: ./ci/start-core.sh
+      - name: Setup rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
       - name: Test
-        run: $HOME/.cargo/bin/cargo test --features test-electrum --no-default-features
-
+        run: cargo test --no-default-features --features ${{ matrix.blockchain.features }} ${{ matrix.blockchain.name }}::bdk_blockchain_tests
+  
   check-wasm:
     name: Check WASM
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-20.04
     env:
       CC: clang-10
       CFLAGS: -I/usr/include
@@ -126,11 +131,11 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
         # Install a recent version of clang that supports wasm32
       - run: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
-      - run: sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" || exit 1
+      - run: sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" || exit 1
       - run: sudo apt-get update || exit 1
-      - run: sudo apt-get install -y clang-10 libc6-dev-i386 || exit 1
+      - run: sudo apt-get install -y libclang-common-10-dev clang-10 libc6-dev-i386 || exit 1
       - name: Set default toolchain
-        run: rustup default stable
+        run: rustup default 1.56.0 # STABLE
       - name: Set profile
         run: rustup set profile minimal
       - name: Add target wasm32
@@ -138,7 +143,7 @@ jobs:
       - name: Update toolchain
         run: rustup update
       - name: Check
-        run: cargo check --target wasm32-unknown-unknown --features esplora --no-default-features
+        run: cargo check --target wasm32-unknown-unknown --features use-esplora-reqwest --no-default-features
 
   fmt:
     name: Rust fmt
@@ -147,12 +152,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set default toolchain
-        run: rustup default stable
+        run: rustup default nightly
       - name: Set profile
         run: rustup set profile minimal
-      - name: Add clippy
+      - name: Add rustfmt
         run: rustup component add rustfmt
       - name: Update toolchain
         run: rustup update
       - name: Check fmt
-        run: cargo fmt --all -- --check
+        run: cargo fmt --all -- --config format_code_in_doc_comments=true --check

.github/workflows/nightly_docs.yml

@@ -17,17 +17,14 @@ jobs:
             ~/.cargo/git
             target
           key: nightly-docs-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
-      - name: Install nightly toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly
-          override: true
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Update toolchain
+        run: rustup update
       - name: Build docs
-        uses: actions-rs/cargo@v1
-        with:
-          command: rustdoc
-          args: --verbose --features=compiler,electrum,esplora,compact_filters,key-value-db,all-keys -- --cfg docsrs -Dwarnings
+        run: cargo rustdoc --verbose --features=compiler,electrum,esplora,ureq,compact_filters,key-value-db,all-keys,sqlite -- --cfg docsrs -Dwarnings
       - name: Upload artifact
         uses: actions/upload-artifact@v2
         with:

CHANGELOG.md

@@ -6,6 +6,103 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [v0.13.0] - [v0.12.0]
+
+- Exposed `get_tx()` method from `Database` to `Wallet`.
+
+## [v0.12.0] - [v0.11.0]
+
+- Activate `miniscript/use-serde` feature to allow consumers of the library to access it via the re-exported `miniscript` crate.
+- Add support for proxies in `EsploraBlockchain`
+- Added `SqliteDatabase` that implements `Database` backed by a sqlite database using `rusqlite` crate.
+
+## [v0.11.0] - [v0.10.0]
+
+- Added `flush` method to the `Database` trait to explicitly flush to disk latest changes on the db.
+
+## [v0.10.0] - [v0.9.0]
+
+- Added `RpcBlockchain` in the `AnyBlockchain` struct to allow using Rpc backend where `AnyBlockchain` is used (eg `bdk-cli`)
+- Removed hard dependency on `tokio`.
+
+### Wallet
+
+- Removed and replaced `set_single_recipient` with more general `drain_to` and replaced `maintain_single_recipient` with `allow_shrinking`.
+
+### Blockchain
+
+- Removed `stop_gap` from `Blockchain` trait and added it to only `ElectrumBlockchain` and `EsploraBlockchain` structs.
+- Added a `ureq` backend for use when not using feature `async-interface` or target WASM. `ureq` is a blocking HTTP client.
+
+## [v0.9.0] - [v0.8.0]
+
+### Wallet
+
+- Added Bitcoin core RPC added as blockchain backend
+- Added a `verify` feature that can be enable to verify the unconfirmed txs we download against the consensus rules
+
+## [v0.8.0] - [v0.7.0]
+
+### Wallet
+- Added an option that must be explicitly enabled to allow signing using non-`SIGHASH_ALL` sighashes (#350)
+#### Changed
+`get_address` now returns an `AddressInfo` struct that includes the index and derefs to `Address`.
+
+## [v0.7.0] - [v0.6.0]
+
+### Policy
+#### Changed
+Removed `fill_satisfaction` method in favor of enum parameter in `extract_policy` method
+
+#### Added
+Timelocks are considered (optionally) in building the `satisfaction` field
+
+### Wallet
+
+- Changed `Wallet::{sign, finalize_psbt}` now take a `&mut psbt` rather than consuming it.
+- Require and validate `non_witness_utxo` for SegWit signatures by default, can be adjusted with `SignOptions`
+- Replace the opt-in builder option `force_non_witness_utxo` with the opposite `only_witness_utxo`. From now on we will provide the `non_witness_utxo`, unless explicitly asked not to.
+
+## [v0.6.0] - [v0.5.1]
+
+### Misc
+#### Changed
+- New minimum supported rust version is 1.46.0
+- Changed `AnyBlockchainConfig` to use serde tagged representation.
+
+### Descriptor
+#### Added
+- Added ability to analyze a `PSBT` to check which and how many signatures are already available
+
+### Wallet
+#### Changed
+- `get_new_address()` refactored to `get_address(AddressIndex::New)` to support different `get_address()` index selection strategies
+
+#### Added
+- Added `get_address(AddressIndex::LastUnused)` which returns the last derived address if it has not been used or if used in a received transaction returns a new address
+- Added `get_address(AddressIndex::Peek(u32))` which returns a derived address for a specified descriptor index but does not change the current index
+- Added `get_address(AddressIndex::Reset(u32))` which returns a derived address for a specified descriptor index and resets current index to the given value
+- Added `get_psbt_input` to create the corresponding psbt input for a local utxo.
+
+#### Fixed
+- Fixed `coin_select` calculation for UTXOs where `value < fee` that caused over-/underflow errors.
+
+## [v0.5.1] - [v0.5.0]
+
+### Misc
+#### Changed
+- Pin `hyper` to `=0.14.4` to make it compile on Rust 1.45
+
+## [v0.5.0] - [v0.4.0]
+
+### Misc
+#### Changed
+- Updated `electrum-client` to version `0.7`
+
+### Wallet
+#### Changed
+- `FeeRate` constructors `from_sat_per_vb` and `default_min_relay_fee` are now `const` functions
+
 ## [v0.4.0] - [v0.3.0]
 
 ### Keys
@@ -50,7 +147,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   type to mark for a missing client.
 - Upgrade `tokio` to `1.0`.
 
-#### Transaction Creation Overhaul
+### Transaction Creation Overhaul
 
 The `TxBuilder` is now created from the `build_tx` or `build_fee_bump` functions on wallet and the
 final transaction is created by calling `finish` on the builder.
@@ -61,6 +158,13 @@ final transaction is created by calling `finish` on the builder.
 - Added `Wallet::get_utxo`
 - Added `Wallet::get_descriptor_for_keychain`
 
+### `add_foreign_utxo`
+
+- Renamed `UTXO` to `LocalUtxo`
+- Added `WeightedUtxo` to replace floating `(UTXO, usize)`.
+- Added `Utxo` enum to incorporate both local utxos and foreign utxos
+- Added `TxBuilder::add_foreign_utxo` which allows adding a utxo external to the wallet.
+
 ### CLI
 #### Changed
 - Remove `cli.rs` module, `cli-utils` feature and `repl.rs` example; moved to new [`bdk-cli`](https://github.com/bitcoindevkit/bdk-cli) repository
@@ -270,8 +374,18 @@ final transaction is created by calling `finish` on the builder.
 - Use `MemoryDatabase` in the compiler example
 - Make the REPL return JSON
 
-[unreleased]: https://github.com/bitcoindevkit/bdk/compare/v0.2.0...HEAD
+[unreleased]: https://github.com/bitcoindevkit/bdk/compare/v0.11.0...HEAD
 [0.1.0-beta.1]: https://github.com/bitcoindevkit/bdk/compare/96c87ea5...0.1.0-beta.1
 [v0.2.0]: https://github.com/bitcoindevkit/bdk/compare/0.1.0-beta.1...v0.2.0
 [v0.3.0]: https://github.com/bitcoindevkit/bdk/compare/v0.2.0...v0.3.0
 [v0.4.0]: https://github.com/bitcoindevkit/bdk/compare/v0.3.0...v0.4.0
+[v0.5.0]: https://github.com/bitcoindevkit/bdk/compare/v0.4.0...v0.5.0
+[v0.5.1]: https://github.com/bitcoindevkit/bdk/compare/v0.5.0...v0.5.1
+[v0.6.0]: https://github.com/bitcoindevkit/bdk/compare/v0.5.1...v0.6.0
+[v0.7.0]: https://github.com/bitcoindevkit/bdk/compare/v0.6.0...v0.7.0
+[v0.8.0]: https://github.com/bitcoindevkit/bdk/compare/v0.7.0...v0.8.0
+[v0.9.0]: https://github.com/bitcoindevkit/bdk/compare/v0.8.0...v0.9.0
+[v0.10.0]: https://github.com/bitcoindevkit/bdk/compare/v0.9.0...v0.10.0
+[v0.11.0]: https://github.com/bitcoindevkit/bdk/compare/v0.10.0...v0.11.0
+[v0.12.0]: https://github.com/bitcoindevkit/bdk/compare/v0.11.0...v0.12.0
+[v0.13.0]: https://github.com/bitcoindevkit/bdk/compare/v0.12.0...v0.13.0

CONTRIBUTING.md

@@ -46,7 +46,7 @@ Every new feature should be covered by functional tests where possible.
 When refactoring, structure your PR to make it easy to review and don't
 hesitate to split it into multiple small, focused PRs.
 
-The Minimal Supported Rust Version is 1.45 (enforced by our CI).
+The Minimal Supported Rust Version is 1.46 (enforced by our CI).
 
 Commits should cover both the issue fixed and the solution's rationale.
 These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind.
@@ -57,6 +57,21 @@ comment suggesting that you're working on it. If someone is already assigned,
 don't hesitate to ask if the assigned party or previous commenters are still
 working on it if it has been awhile.
 
+Deprecation policy
+------------------
+
+Where possible, breaking existing APIs should be avoided. Instead, add new APIs and
+use [`#[deprecated]`](https://github.com/rust-lang/rfcs/blob/master/text/1270-deprecation.md)
+to discourage use of the old one.
+
+Deprecated APIs are typically maintained for one release cycle. In other words, an
+API that has been deprecated with the 0.10 release can be expected to be removed in the
+0.11 release. This allows for smoother upgrades without incurring too much technical
+debt inside this library.
+
+If you deprecated an API as part of a contribution, we encourage you to "own" that API
+and send a follow-up to remove it as part of the next release cycle.
+
 Peer review
 -----------
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk"
-version = "0.4.0"
+version = "0.13.0"
 edition = "2018"
 authors = ["Alekos Filini <alekos.filini@gmail.com>", "Riccardo Casatta <riccardo@casatta.it>"]
 homepage = "https://bitcoindevkit.org"
@@ -9,28 +9,43 @@ documentation = "https://docs.rs/bdk"
 description = "A modern, lightweight, descriptor-based wallet library"
 keywords = ["bitcoin", "wallet", "descriptor", "psbt"]
 readme = "README.md"
-license = "MIT"
+license = "MIT OR Apache-2.0"
 
 [dependencies]
-bdk-macros = "^0.3"
+bdk-macros = "^0.6"
 log = "^0.4"
-miniscript = "5.1"
-bitcoin = { version = "^0.26", features = ["use-serde"] }
+miniscript = { version = "^6.0", features = ["use-serde"] }
+bitcoin = { version = "^0.27", features = ["use-serde", "base64"] }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
 rand = "^0.7"
 
 # Optional dependencies
 sled = { version = "0.34", optional = true }
-electrum-client = { version = "0.6", optional = true }
+electrum-client = { version = "0.8", optional = true }
+rusqlite = { version = "0.25.3", optional = true }
+ahash = { version = "=0.7.4", optional = true }
 reqwest = { version = "0.11", optional = true, features = ["json"] }
+ureq = { version = "~2.2.0", features = ["json"], optional = true }
 futures = { version = "0.3", optional = true }
 async-trait = { version = "0.1", optional = true }
-rocksdb = { version = "0.14", optional = true }
+rocksdb = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
 cc = { version = ">=1.0.64", optional = true }
 socks = { version = "0.3", optional = true }
 lazy_static = { version = "1.4", optional = true }
-tiny-bip39 = { version = "^0.8", optional = true }
+
+# the latest 0.8 version of tiny-bip39 depends on zeroize_derive 1.2 which has MSRV 1.51 and our 
+# MSRV is 1.46, to fix this until we update our MSRV or replace the tiny-bip39 
+# dependency https://github.com/bitcoindevkit/bdk/issues/399 we can only use an older version
+tiny-bip39 = { version = "< 0.8", optional = true }
+# backtrace > 0.3.61 includes object v0.27 which doesn't compile on 1.46. this is used by
+# tiny-bip39
+backtrace = { version = "=0.3.61", optional = true }
+
+bitcoinconsensus = { version = "0.19.0-3", optional = true }
+
+# Needed by bdk_blockchain_tests macro
+bitcoincore-rpc = { version = "0.14", optional = true }
 
 # Platform-specific dependencies
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -44,28 +59,49 @@ rand = { version = "^0.7", features = ["wasm-bindgen"] }
 [features]
 minimal = []
 compiler = ["miniscript/compiler"]
+verify = ["bitcoinconsensus"]
 default = ["key-value-db", "electrum"]
-electrum = ["electrum-client"]
-esplora = ["reqwest", "futures"]
+sqlite = ["rusqlite", "ahash"]
 compact_filters = ["rocksdb", "socks", "lazy_static", "cc"]
 key-value-db = ["sled"]
-async-interface = ["async-trait"]
 all-keys = ["keys-bip39"]
-keys-bip39 = ["tiny-bip39"]
+keys-bip39 = ["tiny-bip39", "backtrace"]
+rpc = ["bitcoincore-rpc"]
+
+# We currently provide mulitple implementations of `Blockchain`, all are
+# blocking except for the `EsploraBlockchain` which can be either async or
+# blocking, depending on the HTTP client in use.
+#
+# - Users wanting asynchronous HTTP calls should enable `async-interface` to get
+#   access to the asynchronous method implementations. Then, if Esplora is wanted,
+#   enable `esplora` AND `reqwest` (`--features=use-esplora-reqwest`).
+# - Users wanting blocking HTTP calls can use any of the other blockchain
+#   implementations (`compact_filters`, `electrum`, or `esplora`). Users wanting to
+#   use Esplora should enable `esplora` AND `ureq`  (`--features=use-esplora-ureq`).
+#
+# WARNING: Please take care with the features below, various combinations will
+# fail to build. We cannot currently build `bdk` with `--all-features`.
+async-interface = ["async-trait"]
+electrum = ["electrum-client"]
+# MUST ALSO USE `--no-default-features`.
+use-esplora-reqwest = ["esplora", "reqwest", "reqwest/socks", "futures"]
+use-esplora-ureq = ["esplora", "ureq", "ureq/socks"]
+# Typical configurations will not need to use `esplora` feature directly.
+esplora = []
+
 
 # Debug/Test features
-debug-proc-macros = ["bdk-macros/debug", "bdk-testutils-macros/debug"]
-test-electrum = ["electrum"]
+test-blockchains = ["bitcoincore-rpc", "electrum-client"]
+test-electrum = ["electrum", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-rpc = ["rpc", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-esplora = ["electrsd/legacy", "electrsd/esplora_a33e97e1", "test-blockchains"]
 test-md-docs = ["electrum"]
 
 [dev-dependencies]
-bdk-testutils = "^0.3"
-bdk-testutils-macros = "^0.3"
-serial_test = "0.4"
 lazy_static = "1.4"
 env_logger = "0.7"
-base64 = "^0.11"
 clap = "2.33"
+electrsd = { version= "0.12", features = ["trigger", "bitcoind_0_21_1"] }
 
 [[example]]
 name = "address_validator"
@@ -79,11 +115,8 @@ path = "examples/compiler.rs"
 required-features = ["compiler"]
 
 [workspace]
-members = ["macros", "testutils", "testutils-macros"]
-
-# Generate docs with nightly to add the "features required" badge
-# https://stackoverflow.com/questions/61417452/how-to-get-a-feature-requirement-tag-in-the-documentation-generated-by-cargo-do
+members = ["macros"]
 [package.metadata.docs.rs]
-features = ["compiler", "electrum", "esplora", "compact_filters", "key-value-db", "all-keys"]
+features = ["compiler", "electrum", "esplora", "ureq", "compact_filters", "rpc", "key-value-db", "sqlite", "all-keys", "verify"]
 # defines the configuration attribute `docsrs`
 rustdoc-args = ["--cfg", "docsrs"]

DEVELOPMENT_CYCLE.md

@@ -20,7 +20,7 @@ As soon as the release is tagged and published, the `release` branch will be mer
 
 ## Making the Release
 
-What follows are notes and procedures that maintaners can refer to when making releases. All the commits and tags must be signed and, ideally, also [timestamped](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md).
+What follows are notes and procedures that maintainers can refer to when making releases. All the commits and tags must be signed and, ideally, also [timestamped](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md).
 
 Pre-`v1.0.0` our "major" releases only affect the "minor" semver value. Accordingly, our "minor" releases will only affect the "patch" value.
 
@@ -32,14 +32,15 @@ Pre-`v1.0.0` our "major" releases only affect the "minor" semver value. Accordin
     - If it's a minor issue you can just fix it in the release branch, since it will be merged back to `master` eventually
     - For bigger issues you can fix them on `master` and then *cherry-pick* the commit to the release branch
 6. Update the changelog with the new release version.
-7. Update `src/lib.rs` with the new version (line ~59)
+7. Update `src/lib.rs` with the new version (line ~43)
 8. On release day, make a commit on the release branch to bump the version to `x.y.z`. The message should be "Bump version to x.y.z".
 9. Add a tag to this commit. The tag name should be `vx.y.z` (for example `v0.5.0`), and the message "Release x.y.z". Make sure the tag is signed, for extra safety use the explicit `--sign` flag.
 10. Push the new commits to the upstream release branch, wait for the CI to finish one last time.
 11. Publish **all** the updated crates to crates.io.
 12. Make a new commit to bump the version value to `x.y.(z+1)-dev`. The message should be "Bump version to x.y.(z+1)-dev".
 13. Merge the release branch back into `master`.
-14. Create the release on GitHub: go to "tags", click on the dots on the right and select "Create Release". Then set the title to `vx.y.z` and write down some brief release notes.
-15. Make sure the new release shows up on crates.io and that the docs are built correctly on docs.rs.
-16. Announce the release on Twitter, Discord and Telegram.
-17. Celebrate :tada:
+14. If the `master` branch contains any unreleased changes to the `bdk-macros` crate, change the `bdk` Cargo.toml `[dependencies]` to point to the local path (ie. `bdk-macros = { path = "./macros"}`)
+15. Create the release on GitHub: go to "tags", click on the dots on the right and select "Create Release". Then set the title to `vx.y.z` and write down some brief release notes.
+16. Make sure the new release shows up on crates.io and that the docs are built correctly on docs.rs.
+17. Announce the release on Twitter, Discord and Telegram.
+18. Celebrate :tada:

LICENSE

@@ -1,21 +1,14 @@
-MIT License
+This software is licensed under [Apache 2.0](LICENSE-APACHE) or
+[MIT](LICENSE-MIT), at your option.
 
-Copyright (c) 2020 Magical Bitcoin
+Some files retain their own copyright notice, however, for full authorship
+information, see version control history.
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+Except as otherwise noted in individual files, all files in this repository are
+licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+http://www.apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
+http://opensource.org/licenses/MIT>, at your option.
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+You may not use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of this software or any files in this repository except in
+accordance with one or both of these licenses.

LICENSE-APACHE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE-MIT

@@ -0,0 +1,16 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -9,11 +9,11 @@
 
   <p>
     <a href="https://crates.io/crates/bdk"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk.svg"/></a>
-    <a href="https://github.com/bitcoindevkit/bdk/blob/master/LICENSE"><img alt="MIT Licensed" src="https://img.shields.io/badge/license-MIT-blue.svg"/></a>
+    <a href="https://github.com/bitcoindevkit/bdk/blob/master/LICENSE"><img alt="MIT or Apache-2.0 Licensed" src="https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
     <a href="https://codecov.io/gh/bitcoindevkit/bdk"><img src="https://codecov.io/gh/bitcoindevkit/bdk/branch/master/graph/badge.svg"/></a>
     <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
-    <a href="https://blog.rust-lang.org/2020/07/16/Rust-1.45.0.html"><img alt="Rustc Version 1.45+" src="https://img.shields.io/badge/rustc-1.45%2B-lightgrey.svg"/></a>
+    <a href="https://blog.rust-lang.org/2020/08/27/Rust-1.46.0.html"><img alt="Rustc Version 1.46+" src="https://img.shields.io/badge/rustc-1.46%2B-lightgrey.svg"/></a>
     <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
 
@@ -67,6 +67,7 @@ fn main() -> Result<(), bdk::Error> {
 
 ```rust
 use bdk::{Wallet, database::MemoryDatabase};
+use bdk::wallet::AddressIndex::New;
 
 fn main() -> Result<(), bdk::Error> {
     let wallet = Wallet::new_offline(
@@ -76,9 +77,9 @@ fn main() -> Result<(), bdk::Error> {
         MemoryDatabase::default(),
     )?;
 
-    println!("Address #0: {}", wallet.get_new_address()?);
-    println!("Address #1: {}", wallet.get_new_address()?);
-    println!("Address #2: {}", wallet.get_new_address()?);
+    println!("Address #0: {}", wallet.get_address(New)?);
+    println!("Address #1: {}", wallet.get_address(New)?);
+    println!("Address #2: {}", wallet.get_address(New)?);
 
     Ok(())
 }
@@ -92,6 +93,7 @@ use bdk::database::MemoryDatabase;
 use bdk::blockchain::{noop_progress, ElectrumBlockchain};
 
 use bdk::electrum_client::Client;
+use bdk::wallet::AddressIndex::New;
 
 use bitcoin::consensus::serialize;
 
@@ -107,7 +109,7 @@ fn main() -> Result<(), bdk::Error> {
 
     wallet.sync(noop_progress(), None)?;
 
-    let send_to = wallet.get_new_address()?;
+    let send_to = wallet.get_address(New)?;
     let (psbt, details) = {
         let mut builder = wallet.build_tx();
         builder
@@ -128,7 +130,7 @@ fn main() -> Result<(), bdk::Error> {
 ### Sign a transaction
 
 ```rust,no_run
-use bdk::{Wallet, database::MemoryDatabase};
+use bdk::{Wallet, SignOptions, database::MemoryDatabase};
 
 use bitcoin::consensus::deserialize;
 
@@ -141,10 +143,46 @@ fn main() -> Result<(), bdk::Error> {
     )?;
 
     let psbt = "...";
-    let psbt = deserialize(&base64::decode(psbt).unwrap())?;
+    let mut psbt = deserialize(&base64::decode(psbt).unwrap())?;
 
-    let (signed_psbt, finalized) = wallet.sign(psbt, None)?;
+    let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
 
     Ok(())
 }
 ```
+
+## Testing
+
+### Unit testing
+
+```
+cargo test
+```
+
+### Integration testing
+
+Integration testing require testing features, for example:
+
+```
+cargo test --features test-electrum
+```
+
+The other options are `test-esplora` or `test-rpc`.
+Note that `electrs` and `bitcoind` binaries are automatically downloaded (on mac and linux), to specify you already have installed binaries you must use `--no-default-features` and provide `BITCOIND_EXE` and `ELECTRS_EXE` as environment variables.
+
+## License
+
+Licensed under either of
+
+ * Apache License, Version 2.0
+   ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+ * MIT license
+   ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+
+at your option.
+
+## Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in the work by you, as defined in the Apache-2.0 license, shall be
+dual licensed as above, without any additional terms or conditions.

ci/start-core.sh

@@ -1,17 +1,14 @@
 #!/usr/bin/env sh
 
 echo "Starting bitcoin node."
-/root/bitcoind -regtest -server -daemon -fallbackfee=0.0002 -rpcuser=admin -rpcpassword=passw -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0
+mkdir $GITHUB_WORKSPACE/.bitcoin
+/root/bitcoind -regtest -server -daemon -datadir=$GITHUB_WORKSPACE/.bitcoin -fallbackfee=0.0002 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -blockfilterindex=1 -peerblockfilters=1
 
 echo "Waiting for bitcoin node."
-until /root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw getblockchaininfo; do
+until /root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin getblockchaininfo; do
     sleep 1
 done
-
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin createwallet $BDK_RPC_WALLET
 echo "Generating 150 bitcoin blocks."
-ADDR=$(/root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw getnewaddress)
-/root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw generatetoaddress 150 $ADDR
-
-echo "Starting electrs node."
-nohup /root/electrs --network regtest --jsonrpc-import &
-sleep 5
+ADDR=$(/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin -rpcwallet=$BDK_RPC_WALLET getnewaddress)
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin generatetoaddress 150 $ADDR

codecov.yaml

@@ -0,0 +1,13 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+        base: auto
+        informational: false
+    patch:
+      default:
+        target: auto
+        threshold: 100%
+        base: auto

examples/address_validator.rs

@@ -1,36 +1,24 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::sync::Arc;
 
 use bdk::bitcoin;
 use bdk::database::MemoryDatabase;
-use bdk::descriptor::HDKeyPaths;
+use bdk::descriptor::HdKeyPaths;
 use bdk::wallet::address_validator::{AddressValidator, AddressValidatorError};
 use bdk::KeychainKind;
 use bdk::Wallet;
 
+use bdk::wallet::AddressIndex::New;
 use bitcoin::hashes::hex::FromHex;
 use bitcoin::util::bip32::Fingerprint;
 use bitcoin::{Network, Script};
@@ -41,7 +29,7 @@ impl AddressValidator for DummyValidator {
     fn validate(
         &self,
         keychain: KeychainKind,
-        hd_keypaths: &HDKeyPaths,
+        hd_keypaths: &HdKeyPaths,
         script: &Script,
     ) -> Result<(), AddressValidatorError> {
         let (_, path) = hd_keypaths
@@ -65,9 +53,9 @@ fn main() -> Result<(), bdk::Error> {
 
     wallet.add_address_validator(Arc::new(DummyValidator));
 
-    wallet.get_new_address()?;
-    wallet.get_new_address()?;
-    wallet.get_new_address()?;
+    wallet.get_address(New)?;
+    wallet.get_address(New)?;
+    wallet.get_address(New)?;
 
     Ok(())
 }

examples/compact_filters_balance.rs

@@ -1,3 +1,14 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
 use bdk::blockchain::compact_filters::*;
 use bdk::blockchain::noop_progress;
 use bdk::database::MemoryDatabase;

examples/compiler.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 extern crate bdk;
 extern crate bitcoin;
@@ -41,6 +28,7 @@ use miniscript::policy::Concrete;
 use miniscript::Descriptor;
 
 use bdk::database::memory::MemoryDatabase;
+use bdk::wallet::AddressIndex::New;
 use bdk::{KeychainKind, Wallet};
 
 fn main() -> Result<(), Box<dyn Error>> {
@@ -82,7 +70,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     let policy_str = matches.value_of("POLICY").unwrap();
     info!("Compiling policy: {}", policy_str);
 
-    let policy = Concrete::<String>::from_str(&policy_str)?;
+    let policy = Concrete::<String>::from_str(policy_str)?;
 
     let descriptor = match matches.value_of("TYPE").unwrap() {
         "sh" => Descriptor::new_sh(policy.compile()?)?,
@@ -97,13 +85,13 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     let network = matches
         .value_of("network")
-        .and_then(|n| Some(Network::from_str(n)))
+        .map(|n| Network::from_str(n))
         .transpose()
         .unwrap()
         .unwrap_or(Network::Testnet);
     let wallet = Wallet::new_offline(&format!("{}", descriptor), None, network, database)?;
 
-    info!("... First address: {}", wallet.get_new_address()?);
+    info!("... First address: {}", wallet.get_address(New)?);
 
     if matches.is_present("parsed_policy") {
         let spending_policy = wallet.policies(KeychainKind::External)?;

macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk-macros"
-version = "0.3.0"
+version = "0.6.0"
 authors = ["Alekos Filini <alekos.filini@gmail.com>"]
 edition = "2018"
 homepage = "https://bitcoindevkit.org"
@@ -8,7 +8,7 @@ repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk-macros"
 description = "Supporting macros for `bdk`"
 keywords = ["bdk"]
-license = "MIT"
+license = "MIT OR Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

macros/src/lib.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 #[macro_use]
 extern crate quote;

src/blockchain/any.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Runtime-checked blockchain types
 //!
@@ -50,9 +37,9 @@
 //! )?;
 //! # }
 //!
-//! # #[cfg(feature = "esplora")]
+//! # #[cfg(all(feature = "esplora", feature = "ureq"))]
 //! # {
-//! let esplora_blockchain = EsploraBlockchain::new("...", None);
+//! let esplora_blockchain = EsploraBlockchain::new("...", 20);
 //! let wallet_esplora: Wallet<AnyBlockchain, _> = Wallet::new(
 //!     "...",
 //!     None,
@@ -73,6 +60,8 @@
 //! # use bdk::blockchain::*;
 //! # use bdk::database::MemoryDatabase;
 //! # use bdk::Wallet;
+//! # #[cfg(all(feature = "esplora", feature = "ureq"))]
+//! # {
 //! let config = serde_json::from_str("...")?;
 //! let blockchain = AnyBlockchain::from_config(&config)?;
 //! let wallet = Wallet::new(
@@ -82,6 +71,7 @@
 //!     MemoryDatabase::default(),
 //!     blockchain,
 //! )?;
+//! # }
 //! # Ok::<(), bdk::Error>(())
 //! ```
 
@@ -107,6 +97,8 @@ macro_rules! impl_inner_method {
             AnyBlockchain::Esplora(inner) => inner.$name( $($args, )* ),
             #[cfg(feature = "compact_filters")]
             AnyBlockchain::CompactFilters(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "rpc")]
+            AnyBlockchain::Rpc(inner) => inner.$name( $($args, )* ),
         }
     }
 }
@@ -129,6 +121,10 @@ pub enum AnyBlockchain {
     #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
     /// Compact filters client
     CompactFilters(compact_filters::CompactFiltersBlockchain),
+    #[cfg(feature = "rpc")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "rpc")))]
+    /// RPC client
+    Rpc(rpc::RpcBlockchain),
 }
 
 #[maybe_async]
@@ -139,31 +135,17 @@ impl Blockchain for AnyBlockchain {
 
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(impl_inner_method!(
-            self,
-            setup,
-            stop_gap,
-            database,
-            progress_update
-        ))
+        maybe_await!(impl_inner_method!(self, setup, database, progress_update))
     }
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(impl_inner_method!(
-            self,
-            sync,
-            stop_gap,
-            database,
-            progress_update
-        ))
+        maybe_await!(impl_inner_method!(self, sync, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
@@ -184,13 +166,43 @@ impl Blockchain for AnyBlockchain {
 impl_from!(electrum::ElectrumBlockchain, AnyBlockchain, Electrum, #[cfg(feature = "electrum")]);
 impl_from!(esplora::EsploraBlockchain, AnyBlockchain, Esplora, #[cfg(feature = "esplora")]);
 impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilters, #[cfg(feature = "compact_filters")]);
+impl_from!(rpc::RpcBlockchain, AnyBlockchain, Rpc, #[cfg(feature = "rpc")]);
 
 /// Type that can contain any of the blockchain configurations defined by the library
 ///
 /// This allows storing a single configuration that can be loaded into an [`AnyBlockchain`]
 /// instance. Wallets that plan to offer users the ability to switch blockchain backend at runtime
 /// will find this particularly useful.
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
+///
+/// This type can be serialized from a JSON object like:
+///
+/// ```
+/// # #[cfg(feature = "electrum")]
+/// # {
+/// use bdk::blockchain::{electrum::ElectrumBlockchainConfig, AnyBlockchainConfig};
+/// let config: AnyBlockchainConfig = serde_json::from_str(
+///     r#"{
+///    "type" : "electrum",
+///    "url" : "ssl://electrum.blockstream.info:50002",
+///    "retry": 2,
+///    "stop_gap": 20
+/// }"#,
+/// )
+/// .unwrap();
+/// assert_eq!(
+///     config,
+///     AnyBlockchainConfig::Electrum(ElectrumBlockchainConfig {
+///         url: "ssl://electrum.blockstream.info:50002".into(),
+///         retry: 2,
+///         socks5: None,
+///         timeout: None,
+///         stop_gap: 20,
+///     })
+/// );
+/// # }
+/// ```
+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, PartialEq)]
+#[serde(tag = "type", rename_all = "snake_case")]
 pub enum AnyBlockchainConfig {
     #[cfg(feature = "electrum")]
     #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
@@ -204,6 +216,10 @@ pub enum AnyBlockchainConfig {
     #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
     /// Compact filters client
     CompactFilters(compact_filters::CompactFiltersBlockchainConfig),
+    #[cfg(feature = "rpc")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "rpc")))]
+    /// RPC client configuration
+    Rpc(rpc::RpcConfig),
 }
 
 impl ConfigurableBlockchain for AnyBlockchain {
@@ -223,6 +239,10 @@ impl ConfigurableBlockchain for AnyBlockchain {
             AnyBlockchainConfig::CompactFilters(inner) => AnyBlockchain::CompactFilters(
                 compact_filters::CompactFiltersBlockchain::from_config(inner)?,
             ),
+            #[cfg(feature = "rpc")]
+            AnyBlockchainConfig::Rpc(inner) => {
+                AnyBlockchain::Rpc(rpc::RpcBlockchain::from_config(inner)?)
+            }
         })
     }
 }
@@ -230,3 +250,4 @@ impl ConfigurableBlockchain for AnyBlockchain {
 impl_from!(electrum::ElectrumBlockchainConfig, AnyBlockchainConfig, Electrum, #[cfg(feature = "electrum")]);
 impl_from!(esplora::EsploraBlockchainConfig, AnyBlockchainConfig, Esplora, #[cfg(feature = "esplora")]);
 impl_from!(compact_filters::CompactFiltersBlockchainConfig, AnyBlockchainConfig, CompactFilters, #[cfg(feature = "compact_filters")]);
+impl_from!(rpc::RpcConfig, AnyBlockchainConfig, Rpc, #[cfg(feature = "rpc")]);

src/blockchain/compact_filters/mod.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Compact Filters
 //!
@@ -83,8 +70,8 @@ mod sync;
 use super::{Blockchain, Capability, ConfigurableBlockchain, Progress};
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{KeychainKind, TransactionDetails, UTXO};
-use crate::FeeRate;
+use crate::types::{KeychainKind, LocalUtxo, TransactionDetails};
+use crate::{ConfirmationTime, FeeRate};
 
 use peer::*;
 use store::*;
@@ -159,7 +146,7 @@ impl CompactFiltersBlockchain {
         database: &mut D,
         tx: &Transaction,
         height: Option<u32>,
-        timestamp: u64,
+        timestamp: Option<u64>,
         internal_max_deriv: &mut Option<u32>,
         external_max_deriv: &mut Option<u32>,
     ) -> Result<(), Error> {
@@ -194,7 +181,7 @@ impl CompactFiltersBlockchain {
                 database.get_path_from_script_pubkey(&output.script_pubkey)?
             {
                 debug!("{} output #{} is mine, adding utxo", tx.txid(), i);
-                updates.set_utxo(&UTXO {
+                updates.set_utxo(&LocalUtxo {
                     outpoint: OutPoint::new(tx.txid(), i as u32),
                     txout: output.clone(),
                     keychain,
@@ -219,9 +206,9 @@ impl CompactFiltersBlockchain {
                 transaction: Some(tx.clone()),
                 received: incoming,
                 sent: outgoing,
-                height,
-                timestamp,
-                fees: inputs_sum.saturating_sub(outputs_sum),
+                confirmation_time: ConfirmationTime::new(height, timestamp),
+                verified: height.is_some(),
+                fee: Some(inputs_sum.saturating_sub(outputs_sum)),
             };
 
             info!("Saving tx {}", tx.txid);
@@ -239,9 +226,9 @@ impl Blockchain for CompactFiltersBlockchain {
         vec![Capability::FullHistory].into_iter().collect()
     }
 
+    #[allow(clippy::mutex_atomic)] // Mutex is easier to understand than a CAS loop.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        _stop_gap: Option<usize>, // TODO: move to electrum and esplora only
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
@@ -249,7 +236,7 @@ impl Blockchain for CompactFiltersBlockchain {
 
         let skip_blocks = self.skip_blocks.unwrap_or(0);
 
-        let cf_sync = Arc::new(CFSync::new(Arc::clone(&self.headers), skip_blocks, 0x00)?);
+        let cf_sync = Arc::new(CfSync::new(Arc::clone(&self.headers), skip_blocks, 0x00)?);
 
         let initial_height = self.headers.get_height()?;
         let total_bundles = (first_peer.get_version().start_height as usize)
@@ -267,7 +254,7 @@ impl Blockchain for CompactFiltersBlockchain {
         let total_cost = headers_cost + filters_cost + PROCESS_BLOCKS_COST;
 
         if let Some(snapshot) = sync::sync_headers(
-            Arc::clone(&first_peer),
+            Arc::clone(first_peer),
             Arc::clone(&self.headers),
             |new_height| {
                 let local_headers_cost =
@@ -288,7 +275,7 @@ impl Blockchain for CompactFiltersBlockchain {
         let buried_height = synced_height.saturating_sub(sync::BURIED_CONFIRMATIONS);
         info!("Synced headers to height: {}", synced_height);
 
-        cf_sync.prepare_sync(Arc::clone(&first_peer))?;
+        cf_sync.prepare_sync(Arc::clone(first_peer))?;
 
         let all_scripts = Arc::new(
             database
@@ -307,7 +294,7 @@ impl Blockchain for CompactFiltersBlockchain {
         let mut threads = Vec::with_capacity(self.peers.len());
         for peer in &self.peers {
             let cf_sync = Arc::clone(&cf_sync);
-            let peer = Arc::clone(&peer);
+            let peer = Arc::clone(peer);
             let headers = Arc::clone(&self.headers);
             let all_scripts = Arc::clone(&all_scripts);
             let last_synced_block = Arc::clone(&last_synced_block);
@@ -376,8 +363,8 @@ impl Blockchain for CompactFiltersBlockchain {
         );
         let mut updates = database.begin_batch();
         for details in database.iter_txs(false)? {
-            match details.height {
-                Some(height) if (height as usize) < last_synced_block => continue,
+            match details.confirmation_time {
+                Some(c) if (c.height as usize) < last_synced_block => continue,
                 _ => updates.del_tx(&details.txid, false)?,
             };
         }
@@ -399,7 +386,7 @@ impl Blockchain for CompactFiltersBlockchain {
                     database,
                     tx,
                     Some(height as u32),
-                    0,
+                    None,
                     &mut internal_max_deriv,
                     &mut external_max_deriv,
                 )?;
@@ -410,7 +397,7 @@ impl Blockchain for CompactFiltersBlockchain {
                 database,
                 tx,
                 None,
-                0,
+                None,
                 &mut internal_max_deriv,
                 &mut external_max_deriv,
             )?;
@@ -468,7 +455,7 @@ impl Blockchain for CompactFiltersBlockchain {
 }
 
 /// Data to connect to a Bitcoin P2P peer
-#[derive(Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
 pub struct BitcoinPeerConfig {
     /// Peer address such as 127.0.0.1:18333
     pub address: String,
@@ -479,7 +466,7 @@ pub struct BitcoinPeerConfig {
 }
 
 /// Configuration for a [`CompactFiltersBlockchain`]
-#[derive(Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
 pub struct CompactFiltersBlockchainConfig {
     /// List of peers to try to connect to for asking headers and filters
     pub peers: Vec<BitcoinPeerConfig>,
@@ -549,11 +536,11 @@ pub enum CompactFiltersError {
     NoPeers,
 
     /// Internal database error
-    DB(rocksdb::Error),
+    Db(rocksdb::Error),
     /// Internal I/O error
-    IO(std::io::Error),
+    Io(std::io::Error),
     /// Invalid BIP158 filter
-    BIP158(bitcoin::util::bip158::Error),
+    Bip158(bitcoin::util::bip158::Error),
     /// Internal system time error
     Time(std::time::SystemTimeError),
 
@@ -569,9 +556,9 @@ impl fmt::Display for CompactFiltersError {
 
 impl std::error::Error for CompactFiltersError {}
 
-impl_error!(rocksdb::Error, DB, CompactFiltersError);
-impl_error!(std::io::Error, IO, CompactFiltersError);
-impl_error!(bitcoin::util::bip158::Error, BIP158, CompactFiltersError);
+impl_error!(rocksdb::Error, Db, CompactFiltersError);
+impl_error!(std::io::Error, Io, CompactFiltersError);
+impl_error!(bitcoin::util::bip158::Error, Bip158, CompactFiltersError);
 impl_error!(std::time::SystemTimeError, Time, CompactFiltersError);
 
 impl From<crate::error::Error> for CompactFiltersError {

src/blockchain/compact_filters/peer.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::collections::HashMap;
 use std::net::{TcpStream, ToSocketAddrs};
@@ -240,12 +227,12 @@ impl Peer {
 
         Ok(Peer {
             writer,
-            reader_thread,
             responses,
+            reader_thread,
             connected,
             mempool,
-            network,
             version,
+            network,
         })
     }
 
@@ -275,7 +262,7 @@ impl Peer {
         let message_resp = {
             let mut lock = responses.write().unwrap();
             let message_resp = lock.entry(wait_for).or_default();
-            Arc::clone(&message_resp)
+            Arc::clone(message_resp)
         };
 
         let (lock, cvar) = &*message_resp;
@@ -392,7 +379,7 @@ impl Peer {
             let message_resp = {
                 let mut lock = reader_thread_responses.write().unwrap();
                 let message_resp = lock.entry(in_message.cmd()).or_default();
-                Arc::clone(&message_resp)
+                Arc::clone(message_resp)
             };
 
             let (lock, cvar) = &*message_resp;

src/blockchain/compact_filters/store.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::convert::TryInto;
 use std::fmt;
@@ -46,6 +33,8 @@ use bitcoin::BlockHash;
 use bitcoin::BlockHeader;
 use bitcoin::Network;
 
+use lazy_static::lazy_static;
+
 use super::CompactFiltersError;
 
 lazy_static! {
@@ -119,7 +108,7 @@ where
     }
 
     fn deserialize(data: &[u8]) -> Result<Self, CompactFiltersError> {
-        Ok(deserialize(data).map_err(|_| CompactFiltersError::DataCorruption)?)
+        deserialize(data).map_err(|_| CompactFiltersError::DataCorruption)
     }
 }
 
@@ -131,7 +120,7 @@ impl Encodable for BundleStatus {
             BundleStatus::Init => {
                 written += 0x00u8.consensus_encode(&mut e)?;
             }
-            BundleStatus::CFHeaders { cf_headers } => {
+            BundleStatus::CfHeaders { cf_headers } => {
                 written += 0x01u8.consensus_encode(&mut e)?;
                 written += VarInt(cf_headers.len() as u64).consensus_encode(&mut e)?;
                 for header in cf_headers {
@@ -182,7 +171,7 @@ impl Decodable for BundleStatus {
                     cf_headers.push(FilterHeader::consensus_decode(&mut d)?);
                 }
 
-                Ok(BundleStatus::CFHeaders { cf_headers })
+                Ok(BundleStatus::CfHeaders { cf_headers })
             }
             0x02 => {
                 let num = VarInt::consensus_decode(&mut d)?;
@@ -436,15 +425,14 @@ impl ChainStore<Full> {
 
         let key = StoreEntry::BlockHeaderIndex(Some(*block_hash)).get_key();
         let data = read_store.get_pinned_cf(cf_handle, key)?;
-        Ok(data
-            .map(|data| {
-                Ok::<_, CompactFiltersError>(usize::from_be_bytes(
-                    data.as_ref()
-                        .try_into()
-                        .map_err(|_| CompactFiltersError::DataCorruption)?,
-                ))
-            })
-            .transpose()?)
+        data.map(|data| {
+            Ok::<_, CompactFiltersError>(usize::from_be_bytes(
+                data.as_ref()
+                    .try_into()
+                    .map_err(|_| CompactFiltersError::DataCorruption)?,
+            ))
+        })
+        .transpose()
     }
 
     pub fn get_block_hash(&self, height: usize) -> Result<Option<BlockHash>, CompactFiltersError> {
@@ -453,13 +441,12 @@ impl ChainStore<Full> {
 
         let key = StoreEntry::BlockHeader(Some(height)).get_key();
         let data = read_store.get_pinned_cf(cf_handle, key)?;
-        Ok(data
-            .map(|data| {
-                let (header, _): (BlockHeader, Uint256) =
-                    deserialize(&data).map_err(|_| CompactFiltersError::DataCorruption)?;
-                Ok::<_, CompactFiltersError>(header.block_hash())
-            })
-            .transpose()?)
+        data.map(|data| {
+            let (header, _): (BlockHeader, Uint256) =
+                deserialize(&data).map_err(|_| CompactFiltersError::DataCorruption)?;
+            Ok::<_, CompactFiltersError>(header.block_hash())
+        })
+        .transpose()
     }
 
     pub fn save_full_block(&self, block: &Block, height: usize) -> Result<(), CompactFiltersError> {
@@ -475,10 +462,10 @@ impl ChainStore<Full> {
         let key = StoreEntry::Block(Some(height)).get_key();
         let opt_block = read_store.get_pinned(key)?;
 
-        Ok(opt_block
+        opt_block
             .map(|data| deserialize(&data))
             .transpose()
-            .map_err(|_| CompactFiltersError::DataCorruption)?)
+            .map_err(|_| CompactFiltersError::DataCorruption)
     }
 
     pub fn delete_blocks_until(&self, height: usize) -> Result<(), CompactFiltersError> {
@@ -565,14 +552,14 @@ impl<T: StoreType> ChainStore<T> {
         let prefix = StoreEntry::BlockHeader(None).get_key();
         let iterator = read_store.prefix_iterator_cf(cf_handle, prefix);
 
-        Ok(iterator
+        iterator
             .last()
             .map(|(_, v)| -> Result<_, CompactFiltersError> {
                 let (header, _): (BlockHeader, Uint256) = SerializeDb::deserialize(&v)?;
 
                 Ok(header.block_hash())
             })
-            .transpose()?)
+            .transpose()
     }
 
     pub fn apply(
@@ -636,26 +623,26 @@ impl<T: StoreType> fmt::Debug for ChainStore<T> {
 
 pub enum BundleStatus {
     Init,
-    CFHeaders { cf_headers: Vec<FilterHeader> },
+    CfHeaders { cf_headers: Vec<FilterHeader> },
     CFilters { cf_filters: Vec<Vec<u8>> },
     Processed { cf_filters: Vec<Vec<u8>> },
     Tip { cf_filters: Vec<Vec<u8>> },
     Pruned,
 }
 
-pub struct CFStore {
+pub struct CfStore {
     store: Arc<RwLock<DB>>,
     filter_type: u8,
 }
 
 type BundleEntry = (BundleStatus, FilterHeader);
 
-impl CFStore {
+impl CfStore {
     pub fn new(
         headers_store: &ChainStore<Full>,
         filter_type: u8,
     ) -> Result<Self, CompactFiltersError> {
-        let cf_store = CFStore {
+        let cf_store = CfStore {
             store: Arc::clone(&headers_store.store),
             filter_type,
         };
@@ -716,11 +703,11 @@ impl CFStore {
 
         // FIXME: we have to filter manually because rocksdb sometimes returns stuff that doesn't
         // have the right prefix
-        Ok(iterator
+        iterator
             .filter(|(k, _)| k.starts_with(&prefix))
             .skip(1)
             .map(|(_, data)| Ok::<_, CompactFiltersError>(BundleEntry::deserialize(&data)?.1))
-            .collect::<Result<_, _>>()?)
+            .collect::<Result<_, _>>()
     }
 
     pub fn replace_checkpoints(
@@ -773,7 +760,7 @@ impl CFStore {
         let cf_headers: Vec<FilterHeader> = filter_hashes
             .into_iter()
             .scan(checkpoint, |prev_header, filter_hash| {
-                let filter_header = filter_hash.filter_header(&prev_header);
+                let filter_header = filter_hash.filter_header(prev_header);
                 *prev_header = filter_header;
 
                 Some(filter_header)
@@ -795,7 +782,7 @@ impl CFStore {
         }
 
         let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
-        let value = (BundleStatus::CFHeaders { cf_headers }, checkpoint);
+        let value = (BundleStatus::CfHeaders { cf_headers }, checkpoint);
 
         read_store.put(key, value.serialize())?;
 
@@ -814,7 +801,7 @@ impl CFStore {
             .zip(headers.into_iter())
             .scan(checkpoint, |prev_header, ((_, filter_content), header)| {
                 let filter = BlockFilter::new(&filter_content);
-                if header != filter.filter_header(&prev_header) {
+                if header != filter.filter_header(prev_header) {
                     return Some(Err(CompactFiltersError::InvalidFilter));
                 }
                 *prev_header = header;

src/blockchain/compact_filters/sync.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::collections::{BTreeMap, HashMap, VecDeque};
 use std::sync::{Arc, Mutex};
@@ -38,22 +25,22 @@ use crate::error::Error;
 
 pub(crate) const BURIED_CONFIRMATIONS: usize = 100;
 
-pub struct CFSync {
+pub struct CfSync {
     headers_store: Arc<ChainStore<Full>>,
-    cf_store: Arc<CFStore>,
+    cf_store: Arc<CfStore>,
     skip_blocks: usize,
     bundles: Mutex<VecDeque<(BundleStatus, FilterHeader, usize)>>,
 }
 
-impl CFSync {
+impl CfSync {
     pub fn new(
         headers_store: Arc<ChainStore<Full>>,
         skip_blocks: usize,
         filter_type: u8,
     ) -> Result<Self, CompactFiltersError> {
-        let cf_store = Arc::new(CFStore::new(&headers_store, filter_type)?);
+        let cf_store = Arc::new(CfStore::new(&headers_store, filter_type)?);
 
-        Ok(CFSync {
+        Ok(CfSync {
             headers_store,
             cf_store,
             skip_blocks,
@@ -164,7 +151,7 @@ impl CFSync {
                     checkpoint,
                     headers_resp.filter_hashes,
                 )? {
-                    BundleStatus::CFHeaders { cf_headers } => cf_headers,
+                    BundleStatus::CfHeaders { cf_headers } => cf_headers,
                     _ => return Err(CompactFiltersError::InvalidResponse),
                 };
 
@@ -184,7 +171,7 @@ impl CFSync {
                     .cf_store
                     .advance_to_cf_filters(index, checkpoint, cf_headers, filters)?;
             }
-            if let BundleStatus::CFHeaders { cf_headers } = status {
+            if let BundleStatus::CfHeaders { cf_headers } = status {
                 log::trace!("status: CFHeaders");
 
                 peer.get_cf_filters(
@@ -218,7 +205,7 @@ impl CFSync {
                     let block_hash = self.headers_store.get_block_hash(height)?.unwrap();
 
                     // TODO: also download random blocks?
-                    if process(&block_hash, &BlockFilter::new(&filter))? {
+                    if process(&block_hash, &BlockFilter::new(filter))? {
                         log::debug!("Downloading block {}", block_hash);
 
                         let block = peer

src/blockchain/electrum.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Electrum
 //!
@@ -46,7 +33,7 @@ use bitcoin::{BlockHeader, Script, Transaction, Txid};
 
 use electrum_client::{Client, ConfigBuilder, ElectrumApi, Socks5Config};
 
-use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
+use self::utils::{ElectrumLikeSync, ElsGetHistoryRes};
 use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
@@ -56,18 +43,17 @@ use crate::FeeRate;
 ///
 /// ## Example
 /// See the [`blockchain::electrum`](crate::blockchain::electrum) module for a usage example.
-pub struct ElectrumBlockchain(Client);
-
-#[cfg(test)]
-#[cfg(feature = "test-electrum")]
-#[bdk_blockchain_tests(crate)]
-fn local_electrs() -> ElectrumBlockchain {
-    ElectrumBlockchain::from(Client::new(&testutils::get_electrum_url()).unwrap())
+pub struct ElectrumBlockchain {
+    client: Client,
+    stop_gap: usize,
 }
 
 impl std::convert::From<Client> for ElectrumBlockchain {
     fn from(client: Client) -> Self {
-        ElectrumBlockchain(client)
+        ElectrumBlockchain {
+            client,
+            stop_gap: 20,
+        }
     }
 }
 
@@ -84,34 +70,33 @@ impl Blockchain for ElectrumBlockchain {
 
     fn setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        self.0
-            .electrum_like_setup(stop_gap, database, progress_update)
+        self.client
+            .electrum_like_setup(self.stop_gap, database, progress_update)
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self.0.transaction_get(txid).map(Option::Some)?)
+        Ok(self.client.transaction_get(txid).map(Option::Some)?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self.0.transaction_broadcast(tx).map(|_| ())?)
+        Ok(self.client.transaction_broadcast(tx).map(|_| ())?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
         // TODO: unsubscribe when added to the client, or is there a better call to use here?
 
         Ok(self
-            .0
+            .client
             .block_headers_subscribe()
             .map(|data| data.height as u32)?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
         Ok(FeeRate::from_btc_per_kvb(
-            self.0.estimate_fee(target)? as f32
+            self.client.estimate_fee(target)? as f32
         ))
     }
 }
@@ -120,7 +105,7 @@ impl ElectrumLikeSync for Client {
     fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
         &self,
         scripts: I,
-    ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error> {
         self.batch_script_get_history(scripts)
             .map(|v| {
                 v.into_iter()
@@ -129,7 +114,7 @@ impl ElectrumLikeSync for Client {
                             .map(
                                 |electrum_client::GetHistoryRes {
                                      height, tx_hash, ..
-                                 }| ELSGetHistoryRes {
+                                 }| ElsGetHistoryRes {
                                     height,
                                     tx_hash,
                                 },
@@ -157,7 +142,7 @@ impl ElectrumLikeSync for Client {
 }
 
 /// Configuration for an [`ElectrumBlockchain`]
-#[derive(Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
 pub struct ElectrumBlockchainConfig {
     /// URL of the Electrum server (such as ElectrumX, Esplora, BWT) may start with `ssl://` or `tcp://` and include a port
     ///
@@ -169,6 +154,8 @@ pub struct ElectrumBlockchainConfig {
     pub retry: u8,
     /// Request timeout (seconds)
     pub timeout: Option<u8>,
+    /// Stop searching addresses for transactions after finding an unused gap of this length
+    pub stop_gap: usize,
 }
 
 impl ConfigurableBlockchain for ElectrumBlockchain {
@@ -182,9 +169,17 @@ impl ConfigurableBlockchain for ElectrumBlockchain {
             .socks5(socks5)?
             .build();
 
-        Ok(ElectrumBlockchain(Client::from_config(
-            config.url.as_str(),
-            electrum_config,
-        )?))
+        Ok(ElectrumBlockchain {
+            client: Client::from_config(config.url.as_str(), electrum_config)?,
+            stop_gap: config.stop_gap,
+        })
+    }
+}
+
+#[cfg(test)]
+#[cfg(feature = "test-electrum")]
+crate::bdk_blockchain_tests! {
+    fn test_instance(test_client: &TestClient) -> ElectrumBlockchain {
+        ElectrumBlockchain::from(Client::new(&test_client.electrsd.electrum_url).unwrap())
     }
 }

src/blockchain/esplora/mod.rs

@@ -0,0 +1,129 @@
+//! Esplora
+//!
+//! This module defines a [`EsploraBlockchain`] struct that can query an Esplora
+//! backend populate the wallet's [database](crate::database::Database) by:
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::esplora::EsploraBlockchain;
+//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", 20);
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! Esplora blockchain can use either `ureq` or `reqwest` for the HTTP client
+//! depending on your needs (blocking or async respectively).
+//!
+//! Please note, to configure the Esplora HTTP client correctly use one of:
+//! Blocking:  --features='esplora,ureq'
+//! Async:     --features='async-interface,esplora,reqwest' --no-default-features
+use std::collections::HashMap;
+use std::fmt;
+use std::io;
+
+use serde::Deserialize;
+
+use bitcoin::consensus;
+use bitcoin::{BlockHash, Txid};
+
+use crate::error::Error;
+use crate::FeeRate;
+
+#[cfg(feature = "reqwest")]
+mod reqwest;
+
+#[cfg(feature = "reqwest")]
+pub use self::reqwest::*;
+
+#[cfg(feature = "ureq")]
+mod ureq;
+
+#[cfg(feature = "ureq")]
+pub use self::ureq::*;
+
+fn into_fee_rate(target: usize, estimates: HashMap<String, f64>) -> Result<FeeRate, Error> {
+    let fee_val = estimates
+        .into_iter()
+        .map(|(k, v)| Ok::<_, std::num::ParseIntError>((k.parse::<usize>()?, v)))
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| Error::Generic(e.to_string()))?
+        .into_iter()
+        .take_while(|(k, _)| k <= &target)
+        .map(|(_, v)| v)
+        .last()
+        .unwrap_or(1.0);
+
+    Ok(FeeRate::from_sat_per_vb(fee_val as f32))
+}
+
+/// Data type used when fetching transaction history from Esplora.
+#[derive(Deserialize)]
+pub struct EsploraGetHistory {
+    txid: Txid,
+    status: EsploraGetHistoryStatus,
+}
+
+#[derive(Deserialize)]
+struct EsploraGetHistoryStatus {
+    block_height: Option<usize>,
+}
+
+/// Errors that can happen during a sync with [`EsploraBlockchain`]
+#[derive(Debug)]
+pub enum EsploraError {
+    /// Error during ureq HTTP request
+    #[cfg(feature = "ureq")]
+    Ureq(::ureq::Error),
+    /// Transport error during the ureq HTTP call
+    #[cfg(feature = "ureq")]
+    UreqTransport(::ureq::Transport),
+    /// Error during reqwest HTTP request
+    #[cfg(feature = "reqwest")]
+    Reqwest(::reqwest::Error),
+    /// HTTP response error
+    HttpResponse(u16),
+    /// IO error during ureq response read
+    Io(io::Error),
+    /// No header found in ureq response
+    NoHeader,
+    /// Invalid number returned
+    Parsing(std::num::ParseIntError),
+    /// Invalid Bitcoin data returned
+    BitcoinEncoding(bitcoin::consensus::encode::Error),
+    /// Invalid Hex data returned
+    Hex(bitcoin::hashes::hex::Error),
+
+    /// Transaction not found
+    TransactionNotFound(Txid),
+    /// Header height not found
+    HeaderHeightNotFound(u32),
+    /// Header hash not found
+    HeaderHashNotFound(BlockHash),
+}
+
+impl fmt::Display for EsploraError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for EsploraError {}
+
+#[cfg(feature = "ureq")]
+impl_error!(::ureq::Error, Ureq, EsploraError);
+#[cfg(feature = "ureq")]
+impl_error!(::ureq::Transport, UreqTransport, EsploraError);
+#[cfg(feature = "reqwest")]
+impl_error!(::reqwest::Error, Reqwest, EsploraError);
+impl_error!(io::Error, Io, EsploraError);
+impl_error!(std::num::ParseIntError, Parsing, EsploraError);
+impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
+impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
+
+#[cfg(test)]
+#[cfg(feature = "test-esplora")]
+crate::bdk_blockchain_tests! {
+    fn test_instance(test_client: &TestClient) -> EsploraBlockchain {
+        EsploraBlockchain::new(&format!("http://{}",test_client.electrsd.esplora_url.as_ref().unwrap()), 20)
+    }
+}

src/blockchain/esplora/reqwest.rs

@@ -1,59 +1,33 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
-//! Esplora
-//!
-//! This module defines a [`Blockchain`] struct that can query an Esplora backend
-//! populate the wallet's [database](crate::database::Database) by
-//!
-//! ## Example
-//!
-//! ```no_run
-//! # use bdk::blockchain::esplora::EsploraBlockchain;
-//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", None);
-//! # Ok::<(), bdk::Error>(())
-//! ```
+//! Esplora by way of `reqwest` HTTP client.
 
 use std::collections::{HashMap, HashSet};
-use std::fmt;
 
-use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
+use bitcoin::consensus::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::{sha256, Hash};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use serde::Deserialize;
+use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
 
-use reqwest::{Client, StatusCode};
+use ::reqwest::{Client, StatusCode};
 
-use bitcoin::consensus::{self, deserialize, serialize};
-use bitcoin::hashes::hex::{FromHex, ToHex};
-use bitcoin::hashes::{sha256, Hash};
-use bitcoin::{BlockHash, BlockHeader, Script, Transaction, Txid};
-
-use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
-use super::*;
+use crate::blockchain::esplora::{EsploraError, EsploraGetHistory};
+use crate::blockchain::utils::{ElectrumLikeSync, ElsGetHistoryRes};
+use crate::blockchain::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::wallet::utils::ChunksIterator;
@@ -75,22 +49,37 @@ struct UrlClient {
 /// ## Example
 /// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
 #[derive(Debug)]
-pub struct EsploraBlockchain(UrlClient);
+pub struct EsploraBlockchain {
+    url_client: UrlClient,
+    stop_gap: usize,
+}
 
 impl std::convert::From<UrlClient> for EsploraBlockchain {
     fn from(url_client: UrlClient) -> Self {
-        EsploraBlockchain(url_client)
+        EsploraBlockchain {
+            url_client,
+            stop_gap: 20,
+        }
     }
 }
 
 impl EsploraBlockchain {
-    /// Create a new instance of the client from a base URL
-    pub fn new(base_url: &str, concurrency: Option<u8>) -> Self {
-        EsploraBlockchain(UrlClient {
-            url: base_url.to_string(),
-            client: Client::new(),
-            concurrency: concurrency.unwrap_or(DEFAULT_CONCURRENT_REQUESTS),
-        })
+    /// Create a new instance of the client from a base URL and `stop_gap`.
+    pub fn new(base_url: &str, stop_gap: usize) -> Self {
+        EsploraBlockchain {
+            url_client: UrlClient {
+                url: base_url.to_string(),
+                client: Client::new(),
+                concurrency: DEFAULT_CONCURRENT_REQUESTS,
+            },
+            stop_gap,
+        }
+    }
+
+    /// Set the concurrency to use when doing batch queries against the Esplora instance.
+    pub fn with_concurrency(mut self, concurrency: u8) -> Self {
+        self.url_client.concurrency = concurrency;
+        self
     }
 }
 
@@ -108,42 +97,29 @@ impl Blockchain for EsploraBlockchain {
 
     fn setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
         maybe_await!(self
-            .0
-            .electrum_like_setup(stop_gap, database, progress_update))
+            .url_client
+            .electrum_like_setup(self.stop_gap, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(await_or_block!(self.0._get_tx(txid))?)
+        Ok(await_or_block!(self.url_client._get_tx(txid))?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(await_or_block!(self.0._broadcast(tx))?)
+        Ok(await_or_block!(self.url_client._broadcast(tx))?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
-        Ok(await_or_block!(self.0._get_height())?)
+        Ok(await_or_block!(self.url_client._get_height())?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
-        let estimates = await_or_block!(self.0._get_fee_estimates())?;
-
-        let fee_val = estimates
-            .into_iter()
-            .map(|(k, v)| Ok::<_, std::num::ParseIntError>((k.parse::<usize>()?, v)))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|e| Error::Generic(e.to_string()))?
-            .into_iter()
-            .take_while(|(k, _)| k <= &target)
-            .map(|(_, v)| v)
-            .last()
-            .unwrap_or(1.0);
-
-        Ok(FeeRate::from_sat_per_vb(fee_val as f32))
+        let estimates = await_or_block!(self.url_client._get_fee_estimates())?;
+        super::into_fee_rate(target, estimates)
     }
 }
 
@@ -223,7 +199,7 @@ impl UrlClient {
     async fn _script_get_history(
         &self,
         script: &Script,
-    ) -> Result<Vec<ELSGetHistoryRes>, EsploraError> {
+    ) -> Result<Vec<ElsGetHistoryRes>, EsploraError> {
         let mut result = Vec::new();
         let scripthash = Self::script_to_scripthash(script);
 
@@ -240,7 +216,7 @@ impl UrlClient {
                 .json::<Vec<EsploraGetHistory>>()
                 .await?
                 .into_iter()
-                .map(|x| ELSGetHistoryRes {
+                .map(|x| ElsGetHistoryRes {
                     tx_hash: x.txid,
                     height: x.status.block_height.unwrap_or(0) as i32,
                 }),
@@ -274,7 +250,7 @@ impl UrlClient {
 
             debug!("... adding {} confirmed transactions", len);
 
-            result.extend(response.into_iter().map(|x| ELSGetHistoryRes {
+            result.extend(response.into_iter().map(|x| ElsGetHistoryRes {
                 tx_hash: x.txid,
                 height: x.status.block_height.unwrap_or(0) as i32,
             }));
@@ -304,126 +280,93 @@ impl ElectrumLikeSync for UrlClient {
     fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
         &self,
         scripts: I,
-    ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
-        let future = async {
-            let mut results = vec![];
-            for chunk in ChunksIterator::new(scripts.into_iter(), self.concurrency as usize) {
-                let mut futs = FuturesOrdered::new();
-                for script in chunk {
-                    futs.push(self._script_get_history(&script));
-                }
-                let partial_results: Vec<Vec<ELSGetHistoryRes>> = futs.try_collect().await?;
-                results.extend(partial_results);
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error> {
+        let mut results = vec![];
+        for chunk in ChunksIterator::new(scripts.into_iter(), self.concurrency as usize) {
+            let mut futs = FuturesOrdered::new();
+            for script in chunk {
+                futs.push(self._script_get_history(script));
             }
-            Ok(stream::iter(results).collect().await)
-        };
-
-        await_or_block!(future)
+            let partial_results: Vec<Vec<ElsGetHistoryRes>> = await_or_block!(futs.try_collect())?;
+            results.extend(partial_results);
+        }
+        Ok(await_or_block!(stream::iter(results).collect()))
     }
 
     fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
         &self,
         txids: I,
     ) -> Result<Vec<Transaction>, Error> {
-        let future = async {
-            let mut results = vec![];
-            for chunk in ChunksIterator::new(txids.into_iter(), self.concurrency as usize) {
-                let mut futs = FuturesOrdered::new();
-                for txid in chunk {
-                    futs.push(self._get_tx_no_opt(&txid));
-                }
-                let partial_results: Vec<Transaction> = futs.try_collect().await?;
-                results.extend(partial_results);
+        let mut results = vec![];
+        for chunk in ChunksIterator::new(txids.into_iter(), self.concurrency as usize) {
+            let mut futs = FuturesOrdered::new();
+            for txid in chunk {
+                futs.push(self._get_tx_no_opt(txid));
             }
-            Ok(stream::iter(results).collect().await)
-        };
-
-        await_or_block!(future)
+            let partial_results: Vec<Transaction> = await_or_block!(futs.try_collect())?;
+            results.extend(partial_results);
+        }
+        Ok(await_or_block!(stream::iter(results).collect()))
     }
 
     fn els_batch_block_header<I: IntoIterator<Item = u32>>(
         &self,
         heights: I,
     ) -> Result<Vec<BlockHeader>, Error> {
-        let future = async {
-            let mut results = vec![];
-            for chunk in ChunksIterator::new(heights.into_iter(), self.concurrency as usize) {
-                let mut futs = FuturesOrdered::new();
-                for height in chunk {
-                    futs.push(self._get_header(height));
-                }
-                let partial_results: Vec<BlockHeader> = futs.try_collect().await?;
-                results.extend(partial_results);
+        let mut results = vec![];
+        for chunk in ChunksIterator::new(heights.into_iter(), self.concurrency as usize) {
+            let mut futs = FuturesOrdered::new();
+            for height in chunk {
+                futs.push(self._get_header(height));
             }
-            Ok(stream::iter(results).collect().await)
-        };
-
-        await_or_block!(future)
+            let partial_results: Vec<BlockHeader> = await_or_block!(futs.try_collect())?;
+            results.extend(partial_results);
+        }
+        Ok(await_or_block!(stream::iter(results).collect()))
     }
 }
 
-#[derive(Deserialize)]
-struct EsploraGetHistoryStatus {
-    block_height: Option<usize>,
-}
-
-#[derive(Deserialize)]
-struct EsploraGetHistory {
-    txid: Txid,
-    status: EsploraGetHistoryStatus,
-}
-
 /// Configuration for an [`EsploraBlockchain`]
-#[derive(Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
 pub struct EsploraBlockchainConfig {
     /// Base URL of the esplora service
     ///
     /// eg. `https://blockstream.info/api/`
     pub base_url: String,
+    /// Optional URL of the proxy to use to make requests to the Esplora server
+    ///
+    /// The string should be formatted as: `<protocol>://<user>:<password>@host:<port>`.
+    ///
+    /// Note that the format of this value and the supported protocols change slightly between the
+    /// sync version of esplora (using `ureq`) and the async version (using `reqwest`). For more
+    /// details check with the documentation of the two crates. Both of them are compiled with
+    /// the `socks` feature enabled.
+    ///
+    /// The proxy is ignored when targeting `wasm32`.
+    pub proxy: Option<String>,
     /// Number of parallel requests sent to the esplora service (default: 4)
     pub concurrency: Option<u8>,
+    /// Stop searching addresses for transactions after finding an unused gap of this length.
+    pub stop_gap: usize,
 }
 
 impl ConfigurableBlockchain for EsploraBlockchain {
     type Config = EsploraBlockchainConfig;
 
     fn from_config(config: &Self::Config) -> Result<Self, Error> {
-        Ok(EsploraBlockchain::new(
-            config.base_url.as_str(),
-            config.concurrency,
-        ))
+        let map_e = |e: reqwest::Error| Error::Esplora(Box::new(e.into()));
+
+        let mut blockchain = EsploraBlockchain::new(config.base_url.as_str(), config.stop_gap);
+        if let Some(concurrency) = config.concurrency {
+            blockchain.url_client.concurrency = concurrency;
+        }
+        #[cfg(not(target_arch = "wasm32"))]
+        if let Some(proxy) = &config.proxy {
+            blockchain.url_client.client = Client::builder()
+                .proxy(reqwest::Proxy::all(proxy).map_err(map_e)?)
+                .build()
+                .map_err(map_e)?;
+        }
+        Ok(blockchain)
     }
 }
-
-/// Errors that can happen during a sync with [`EsploraBlockchain`]
-#[derive(Debug)]
-pub enum EsploraError {
-    /// Error with the HTTP call
-    Reqwest(reqwest::Error),
-    /// Invalid number returned
-    Parsing(std::num::ParseIntError),
-    /// Invalid Bitcoin data returned
-    BitcoinEncoding(bitcoin::consensus::encode::Error),
-    /// Invalid Hex data returned
-    Hex(bitcoin::hashes::hex::Error),
-
-    /// Transaction not found
-    TransactionNotFound(Txid),
-    /// Header height not found
-    HeaderHeightNotFound(u32),
-    /// Header hash not found
-    HeaderHashNotFound(BlockHash),
-}
-
-impl fmt::Display for EsploraError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
-
-impl std::error::Error for EsploraError {}
-
-impl_error!(reqwest::Error, Reqwest, EsploraError);
-impl_error!(std::num::ParseIntError, Parsing, EsploraError);
-impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
-impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);

src/blockchain/esplora/ureq.rs

@@ -0,0 +1,398 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Esplora by way of `ureq` HTTP client.
+
+use std::collections::{HashMap, HashSet};
+use std::io;
+use std::io::Read;
+use std::time::Duration;
+
+#[allow(unused_imports)]
+use log::{debug, error, info, trace};
+
+use ureq::{Agent, Proxy, Response};
+
+use bitcoin::consensus::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::{sha256, Hash};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
+
+use crate::blockchain::esplora::{EsploraError, EsploraGetHistory};
+use crate::blockchain::utils::{ElectrumLikeSync, ElsGetHistoryRes};
+use crate::blockchain::*;
+use crate::database::BatchDatabase;
+use crate::error::Error;
+use crate::FeeRate;
+
+#[derive(Debug)]
+struct UrlClient {
+    url: String,
+    agent: Agent,
+}
+
+/// Structure that implements the logic to sync with Esplora
+///
+/// ## Example
+/// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
+#[derive(Debug)]
+pub struct EsploraBlockchain {
+    url_client: UrlClient,
+    stop_gap: usize,
+}
+
+impl std::convert::From<UrlClient> for EsploraBlockchain {
+    fn from(url_client: UrlClient) -> Self {
+        EsploraBlockchain {
+            url_client,
+            stop_gap: 20,
+        }
+    }
+}
+
+impl EsploraBlockchain {
+    /// Create a new instance of the client from a base URL and the `stop_gap`.
+    pub fn new(base_url: &str, stop_gap: usize) -> Self {
+        EsploraBlockchain {
+            url_client: UrlClient {
+                url: base_url.to_string(),
+                agent: Agent::new(),
+            },
+            stop_gap,
+        }
+    }
+
+    /// Set the inner `ureq` agent.
+    pub fn with_agent(mut self, agent: Agent) -> Self {
+        self.url_client.agent = agent;
+        self
+    }
+}
+
+impl Blockchain for EsploraBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        vec![
+            Capability::FullHistory,
+            Capability::GetAnyTx,
+            Capability::AccurateFees,
+        ]
+        .into_iter()
+        .collect()
+    }
+
+    fn setup<D: BatchDatabase, P: Progress>(
+        &self,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        self.url_client
+            .electrum_like_setup(self.stop_gap, database, progress_update)
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(self.url_client._get_tx(txid)?)
+    }
+
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        let _txid = self.url_client._broadcast(tx)?;
+        Ok(())
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(self.url_client._get_height()?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        let estimates = self.url_client._get_fee_estimates()?;
+        super::into_fee_rate(target, estimates)
+    }
+}
+
+impl UrlClient {
+    fn script_to_scripthash(script: &Script) -> String {
+        sha256::Hash::hash(script.as_bytes()).into_inner().to_hex()
+    }
+
+    fn _get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/tx/{}/raw", self.url, txid))
+            .call();
+
+        match resp {
+            Ok(resp) => Ok(Some(deserialize(&into_bytes(resp)?)?)),
+            Err(ureq::Error::Status(code, _)) => {
+                if is_status_not_found(code) {
+                    return Ok(None);
+                }
+                Err(EsploraError::HttpResponse(code))
+            }
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _get_tx_no_opt(&self, txid: &Txid) -> Result<Transaction, EsploraError> {
+        match self._get_tx(txid) {
+            Ok(Some(tx)) => Ok(tx),
+            Ok(None) => Err(EsploraError::TransactionNotFound(*txid)),
+            Err(e) => Err(e),
+        }
+    }
+
+    fn _get_header(&self, block_height: u32) -> Result<BlockHeader, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/block-height/{}", self.url, block_height))
+            .call();
+
+        let bytes = match resp {
+            Ok(resp) => Ok(into_bytes(resp)?),
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }?;
+
+        let hash = std::str::from_utf8(&bytes)
+            .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?;
+
+        let resp = self
+            .agent
+            .get(&format!("{}/block/{}/header", self.url, hash))
+            .call();
+
+        match resp {
+            Ok(resp) => Ok(deserialize(&Vec::from_hex(&resp.into_string()?)?)?),
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
+        let resp = self
+            .agent
+            .post(&format!("{}/tx", self.url))
+            .send_string(&serialize(transaction).to_hex());
+
+        match resp {
+            Ok(_) => Ok(()), // We do not return the txid?
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _get_height(&self) -> Result<u32, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/blocks/tip/height", self.url))
+            .call();
+
+        match resp {
+            Ok(resp) => Ok(resp.into_string()?.parse()?),
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }
+    }
+
+    fn _script_get_history(&self, script: &Script) -> Result<Vec<ElsGetHistoryRes>, EsploraError> {
+        let mut result = Vec::new();
+        let scripthash = Self::script_to_scripthash(script);
+
+        // Add the unconfirmed transactions first
+
+        let resp = self
+            .agent
+            .get(&format!(
+                "{}/scripthash/{}/txs/mempool",
+                self.url, scripthash
+            ))
+            .call();
+
+        let v = match resp {
+            Ok(resp) => {
+                let v: Vec<EsploraGetHistory> = resp.into_json()?;
+                Ok(v)
+            }
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }?;
+
+        result.extend(v.into_iter().map(|x| ElsGetHistoryRes {
+            tx_hash: x.txid,
+            height: x.status.block_height.unwrap_or(0) as i32,
+        }));
+
+        debug!(
+            "Found {} mempool txs for {} - {:?}",
+            result.len(),
+            scripthash,
+            script
+        );
+
+        // Then go through all the pages of confirmed transactions
+        let mut last_txid = String::new();
+        loop {
+            let resp = self
+                .agent
+                .get(&format!(
+                    "{}/scripthash/{}/txs/chain/{}",
+                    self.url, scripthash, last_txid
+                ))
+                .call();
+
+            let v = match resp {
+                Ok(resp) => {
+                    let v: Vec<EsploraGetHistory> = resp.into_json()?;
+                    Ok(v)
+                }
+                Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+                Err(e) => Err(EsploraError::Ureq(e)),
+            }?;
+
+            let len = v.len();
+            if let Some(elem) = v.last() {
+                last_txid = elem.txid.to_hex();
+            }
+
+            debug!("... adding {} confirmed transactions", len);
+
+            result.extend(v.into_iter().map(|x| ElsGetHistoryRes {
+                tx_hash: x.txid,
+                height: x.status.block_height.unwrap_or(0) as i32,
+            }));
+
+            if len < 25 {
+                break;
+            }
+        }
+
+        Ok(result)
+    }
+
+    fn _get_fee_estimates(&self) -> Result<HashMap<String, f64>, EsploraError> {
+        let resp = self
+            .agent
+            .get(&format!("{}/fee-estimates", self.url,))
+            .call();
+
+        let map = match resp {
+            Ok(resp) => {
+                let map: HashMap<String, f64> = resp.into_json()?;
+                Ok(map)
+            }
+            Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)),
+            Err(e) => Err(EsploraError::Ureq(e)),
+        }?;
+
+        Ok(map)
+    }
+}
+
+fn is_status_not_found(status: u16) -> bool {
+    status == 404
+}
+
+fn into_bytes(resp: Response) -> Result<Vec<u8>, io::Error> {
+    const BYTES_LIMIT: usize = 10 * 1_024 * 1_024;
+
+    let mut buf: Vec<u8> = vec![];
+    resp.into_reader()
+        .take((BYTES_LIMIT + 1) as u64)
+        .read_to_end(&mut buf)?;
+    if buf.len() > BYTES_LIMIT {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            "response too big for into_bytes",
+        ));
+    }
+
+    Ok(buf)
+}
+
+impl ElectrumLikeSync for UrlClient {
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
+        &self,
+        scripts: I,
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error> {
+        let mut results = vec![];
+        for script in scripts.into_iter() {
+            let v = self._script_get_history(script)?;
+            results.push(v);
+        }
+        Ok(results)
+    }
+
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
+        &self,
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut results = vec![];
+        for txid in txids.into_iter() {
+            let tx = self._get_tx_no_opt(txid)?;
+            results.push(tx);
+        }
+        Ok(results)
+    }
+
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        let mut results = vec![];
+        for height in heights.into_iter() {
+            let header = self._get_header(height)?;
+            results.push(header);
+        }
+        Ok(results)
+    }
+}
+
+/// Configuration for an [`EsploraBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
+pub struct EsploraBlockchainConfig {
+    /// Base URL of the esplora service eg. `https://blockstream.info/api/`
+    pub base_url: String,
+    /// Optional URL of the proxy to use to make requests to the Esplora server
+    ///
+    /// The string should be formatted as: `<protocol>://<user>:<password>@host:<port>`.
+    ///
+    /// Note that the format of this value and the supported protocols change slightly between the
+    /// sync version of esplora (using `ureq`) and the async version (using `reqwest`). For more
+    /// details check with the documentation of the two crates. Both of them are compiled with
+    /// the `socks` feature enabled.
+    ///
+    /// The proxy is ignored when targeting `wasm32`.
+    pub proxy: Option<String>,
+    /// Socket read timeout.
+    pub timeout_read: u64,
+    /// Socket write timeout.
+    pub timeout_write: u64,
+    /// Stop searching addresses for transactions after finding an unused gap of this length.
+    pub stop_gap: usize,
+}
+
+impl ConfigurableBlockchain for EsploraBlockchain {
+    type Config = EsploraBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let mut agent_builder = ureq::AgentBuilder::new()
+            .timeout_read(Duration::from_secs(config.timeout_read))
+            .timeout_write(Duration::from_secs(config.timeout_write));
+
+        if let Some(proxy) = &config.proxy {
+            agent_builder = agent_builder
+                .proxy(Proxy::new(proxy).map_err(|e| Error::Esplora(Box::new(e.into())))?);
+        }
+
+        Ok(
+            EsploraBlockchain::new(config.base_url.as_str(), config.stop_gap)
+                .with_agent(agent_builder.build()),
+        )
+    }
+}

src/blockchain/mod.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Blockchain backends
 //!
@@ -43,9 +30,19 @@ use crate::FeeRate;
 #[cfg(any(feature = "electrum", feature = "esplora"))]
 pub(crate) mod utils;
 
-#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+#[cfg(any(
+    feature = "electrum",
+    feature = "esplora",
+    feature = "compact_filters",
+    feature = "rpc"
+))]
 pub mod any;
-#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+#[cfg(any(
+    feature = "electrum",
+    feature = "esplora",
+    feature = "compact_filters",
+    feature = "rpc"
+))]
 pub use any::{AnyBlockchain, AnyBlockchainConfig};
 
 #[cfg(feature = "electrum")]
@@ -56,6 +53,14 @@ pub use self::electrum::ElectrumBlockchain;
 #[cfg(feature = "electrum")]
 pub use self::electrum::ElectrumBlockchainConfig;
 
+#[cfg(feature = "rpc")]
+#[cfg_attr(docsrs, doc(cfg(feature = "rpc")))]
+pub mod rpc;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcBlockchain;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcConfig;
+
 #[cfg(feature = "esplora")]
 #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
 pub mod esplora;
@@ -65,6 +70,7 @@ pub use self::esplora::EsploraBlockchain;
 #[cfg(feature = "compact_filters")]
 #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
 pub mod compact_filters;
+
 #[cfg(feature = "compact_filters")]
 pub use self::compact_filters::CompactFiltersBlockchain;
 
@@ -97,7 +103,6 @@ pub trait Blockchain {
     /// [`Blockchain::sync`] defaults to calling this internally if not overridden.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error>;
@@ -122,11 +127,10 @@ pub trait Blockchain {
     /// [`BatchOperations::del_utxo`]: crate::database::BatchOperations::del_utxo
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.setup(stop_gap, database, progress_update))
+        maybe_await!(self.setup(database, progress_update))
     }
 
     /// Fetch a transaction from the blockchain given its txid
@@ -179,7 +183,7 @@ impl Progress for Sender<ProgressData> {
 }
 
 /// Type that implements [`Progress`] and drops every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct NoopProgress;
 
 /// Create a new instance of [`NoopProgress`]
@@ -194,10 +198,10 @@ impl Progress for NoopProgress {
 }
 
 /// Type that implements [`Progress`] and logs at level `INFO` every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct LogProgress;
 
-/// Create a nwe instance of [`LogProgress`]
+/// Create a new instance of [`LogProgress`]
 pub fn log_progress() -> LogProgress {
     LogProgress
 }
@@ -222,20 +226,18 @@ impl<T: Blockchain> Blockchain for Arc<T> {
 
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.deref().setup(stop_gap, database, progress_update))
+        maybe_await!(self.deref().setup(database, progress_update))
     }
 
     fn sync<D: BatchDatabase, P: 'static + Progress>(
         &self,
-        stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        maybe_await!(self.deref().sync(stop_gap, database, progress_update))
+        maybe_await!(self.deref().sync(database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {

src/blockchain/rpc.rs

@@ -0,0 +1,477 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Riccardo Casatta <riccardo@casatta.it>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Rpc Blockchain
+//!
+//! Backend that gets blockchain data from Bitcoin Core RPC
+//!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::{RpcConfig, RpcBlockchain, ConfigurableBlockchain, rpc::Auth};
+//! let config = RpcConfig {
+//!     url: "127.0.0.1:18332".to_string(),
+//!     auth: Auth::Cookie {
+//!         file: "/home/user/.bitcoin/.cookie".into(),
+//!     },
+//!     network: bdk::bitcoin::Network::Testnet,
+//!     wallet_name: "wallet_name".to_string(),
+//!     skip_blocks: None,
+//! };
+//! let blockchain = RpcBlockchain::from_config(&config);
+//! ```
+
+use crate::bitcoin::consensus::deserialize;
+use crate::bitcoin::{Address, Network, OutPoint, Transaction, TxOut, Txid};
+use crate::blockchain::{Blockchain, Capability, ConfigurableBlockchain, Progress};
+use crate::database::{BatchDatabase, DatabaseUtils};
+use crate::descriptor::{get_checksum, IntoWalletDescriptor};
+use crate::wallet::utils::SecpCtx;
+use crate::{ConfirmationTime, Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails};
+use bitcoincore_rpc::json::{
+    GetAddressInfoResultLabel, ImportMultiOptions, ImportMultiRequest,
+    ImportMultiRequestScriptPubkey, ImportMultiRescanSince,
+};
+use bitcoincore_rpc::jsonrpc::serde_json::Value;
+use bitcoincore_rpc::Auth as RpcAuth;
+use bitcoincore_rpc::{Client, RpcApi};
+use log::debug;
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+use std::str::FromStr;
+
+/// The main struct for RPC backend implementing the [crate::blockchain::Blockchain] trait
+#[derive(Debug)]
+pub struct RpcBlockchain {
+    /// Rpc client to the node, includes the wallet name
+    client: Client,
+    /// Network used
+    network: Network,
+    /// Blockchain capabilities, cached here at startup
+    capabilities: HashSet<Capability>,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    skip_blocks: Option<u32>,
+
+    /// This is a fixed Address used as a hack key to store information on the node
+    _storage_address: Address,
+}
+
+/// RpcBlockchain configuration options
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+pub struct RpcConfig {
+    /// The bitcoin node url
+    pub url: String,
+    /// The bitcoin node authentication mechanism
+    pub auth: Auth,
+    /// The network we are using (it will be checked the bitcoin node network matches this)
+    pub network: Network,
+    /// The wallet name in the bitcoin node, consider using [wallet_name_from_descriptor] for this
+    pub wallet_name: String,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    pub skip_blocks: Option<u32>,
+}
+
+/// This struct is equivalent to [bitcoincore_rpc::Auth] but it implements [serde::Serialize]
+/// To be removed once upstream equivalent is implementing Serialize (json serialization format
+/// should be the same), see [rust-bitcoincore-rpc/pull/181](https://github.com/rust-bitcoin/rust-bitcoincore-rpc/pull/181)
+#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(untagged)]
+pub enum Auth {
+    /// None authentication
+    None,
+    /// Authentication with username and password, usually [Auth::Cookie] should be preferred
+    UserPass {
+        /// Username
+        username: String,
+        /// Password
+        password: String,
+    },
+    /// Authentication with a cookie file
+    Cookie {
+        /// Cookie file
+        file: PathBuf,
+    },
+}
+
+impl From<Auth> for RpcAuth {
+    fn from(auth: Auth) -> Self {
+        match auth {
+            Auth::None => RpcAuth::None,
+            Auth::UserPass { username, password } => RpcAuth::UserPass(username, password),
+            Auth::Cookie { file } => RpcAuth::CookieFile(file),
+        }
+    }
+}
+
+impl RpcBlockchain {
+    fn get_node_synced_height(&self) -> Result<u32, Error> {
+        let info = self.client.get_address_info(&self._storage_address)?;
+        if let Some(GetAddressInfoResultLabel::Simple(label)) = info.labels.first() {
+            Ok(label
+                .parse::<u32>()
+                .unwrap_or_else(|_| self.skip_blocks.unwrap_or(0)))
+        } else {
+            Ok(self.skip_blocks.unwrap_or(0))
+        }
+    }
+
+    /// Set the synced height in the core node by using a label of a fixed address so that
+    /// another client with the same descriptor doesn't rescan the blockchain
+    fn set_node_synced_height(&self, height: u32) -> Result<(), Error> {
+        Ok(self
+            .client
+            .set_label(&self._storage_address, &height.to_string())?)
+    }
+}
+
+impl Blockchain for RpcBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        self.capabilities.clone()
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        let mut scripts_pubkeys = database.iter_script_pubkeys(Some(KeychainKind::External))?;
+        scripts_pubkeys.extend(database.iter_script_pubkeys(Some(KeychainKind::Internal))?);
+        debug!(
+            "importing {} script_pubkeys (some maybe already imported)",
+            scripts_pubkeys.len()
+        );
+        let requests: Vec<_> = scripts_pubkeys
+            .iter()
+            .map(|s| ImportMultiRequest {
+                timestamp: ImportMultiRescanSince::Timestamp(0),
+                script_pubkey: Some(ImportMultiRequestScriptPubkey::Script(s)),
+                watchonly: Some(true),
+                ..Default::default()
+            })
+            .collect();
+        let options = ImportMultiOptions {
+            rescan: Some(false),
+        };
+        // Note we use import_multi because as of bitcoin core 0.21.0 many descriptors are not supported
+        // https://bitcoindevkit.org/descriptors/#compatibility-matrix
+        //TODO maybe convenient using import_descriptor for compatible descriptor and import_multi as fallback
+        self.client.import_multi(&requests, Some(&options))?;
+
+        loop {
+            let current_height = self.get_height()?;
+
+            // min because block invalidate may cause height to go down
+            let node_synced = self.get_node_synced_height()?.min(current_height);
+
+            let sync_up_to = node_synced.saturating_add(10_000).min(current_height);
+
+            debug!("rescan_blockchain from:{} to:{}", node_synced, sync_up_to);
+            self.client
+                .rescan_blockchain(Some(node_synced as usize), Some(sync_up_to as usize))?;
+            progress_update.update((sync_up_to as f32) / (current_height as f32), None)?;
+
+            self.set_node_synced_height(sync_up_to)?;
+
+            if sync_up_to == current_height {
+                break;
+            }
+        }
+
+        self.sync(database, progress_update)
+    }
+
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        db: &mut D,
+        _progress_update: P,
+    ) -> Result<(), Error> {
+        let mut indexes = HashMap::new();
+        for keykind in &[KeychainKind::External, KeychainKind::Internal] {
+            indexes.insert(*keykind, db.get_last_index(*keykind)?.unwrap_or(0));
+        }
+
+        let mut known_txs: HashMap<_, _> = db
+            .iter_txs(true)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let known_utxos: HashSet<_> = db.iter_utxos()?.into_iter().collect();
+
+        //TODO list_since_blocks would be more efficient
+        let current_utxo = self
+            .client
+            .list_unspent(Some(0), None, None, Some(true), None)?;
+        debug!("current_utxo len {}", current_utxo.len());
+
+        //TODO supported up to 1_000 txs, should use since_blocks or do paging
+        let list_txs = self
+            .client
+            .list_transactions(None, Some(1_000), None, Some(true))?;
+        let mut list_txs_ids = HashSet::new();
+
+        for tx_result in list_txs.iter().filter(|t| {
+            // list_txs returns all conflicting tx we want to
+            // filter out replaced tx => unconfirmed and not in the mempool
+            t.info.confirmations > 0 || self.client.get_mempool_entry(&t.info.txid).is_ok()
+        }) {
+            let txid = tx_result.info.txid;
+            list_txs_ids.insert(txid);
+            if let Some(mut known_tx) = known_txs.get_mut(&txid) {
+                let confirmation_time =
+                    ConfirmationTime::new(tx_result.info.blockheight, tx_result.info.blocktime);
+                if confirmation_time != known_tx.confirmation_time {
+                    // reorg may change tx height
+                    debug!(
+                        "updating tx({}) confirmation time to: {:?}",
+                        txid, confirmation_time
+                    );
+                    known_tx.confirmation_time = confirmation_time;
+                    db.set_tx(known_tx)?;
+                }
+            } else {
+                //TODO check there is already the raw tx in db?
+                let tx_result = self.client.get_transaction(&txid, Some(true))?;
+                let tx: Transaction = deserialize(&tx_result.hex)?;
+                let mut received = 0u64;
+                let mut sent = 0u64;
+                for output in tx.output.iter() {
+                    if let Ok(Some((kind, index))) =
+                        db.get_path_from_script_pubkey(&output.script_pubkey)
+                    {
+                        if index > *indexes.get(&kind).unwrap() {
+                            indexes.insert(kind, index);
+                        }
+                        received += output.value;
+                    }
+                }
+
+                for input in tx.input.iter() {
+                    if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+                        sent += previous_output.value;
+                    }
+                }
+
+                let td = TransactionDetails {
+                    transaction: Some(tx),
+                    txid: tx_result.info.txid,
+                    confirmation_time: ConfirmationTime::new(
+                        tx_result.info.blockheight,
+                        tx_result.info.blocktime,
+                    ),
+                    received,
+                    sent,
+                    fee: tx_result.fee.map(|f| f.as_sat().abs() as u64),
+                    verified: true,
+                };
+                debug!(
+                    "saving tx: {} tx_result.fee:{:?} td.fees:{:?}",
+                    td.txid, tx_result.fee, td.fee
+                );
+                db.set_tx(&td)?;
+            }
+        }
+
+        for known_txid in known_txs.keys() {
+            if !list_txs_ids.contains(known_txid) {
+                debug!("removing tx: {}", known_txid);
+                db.del_tx(known_txid, false)?;
+            }
+        }
+
+        let current_utxos: HashSet<_> = current_utxo
+            .into_iter()
+            .map(|u| {
+                Ok(LocalUtxo {
+                    outpoint: OutPoint::new(u.txid, u.vout),
+                    keychain: db
+                        .get_path_from_script_pubkey(&u.script_pub_key)?
+                        .ok_or(Error::TransactionNotFound)?
+                        .0,
+                    txout: TxOut {
+                        value: u.amount.as_sat(),
+                        script_pubkey: u.script_pub_key,
+                    },
+                })
+            })
+            .collect::<Result<_, Error>>()?;
+
+        let spent: HashSet<_> = known_utxos.difference(&current_utxos).collect();
+        for s in spent {
+            debug!("removing utxo: {:?}", s);
+            db.del_utxo(&s.outpoint)?;
+        }
+        let received: HashSet<_> = current_utxos.difference(&known_utxos).collect();
+        for s in received {
+            debug!("adding utxo: {:?}", s);
+            db.set_utxo(s)?;
+        }
+
+        for (keykind, index) in indexes {
+            debug!("{:?} max {}", keykind, index);
+            db.set_last_index(keykind, index)?;
+        }
+
+        Ok(())
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(Some(self.client.get_raw_transaction(txid, None)?))
+    }
+
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        Ok(self.client.send_raw_transaction(tx).map(|_| ())?)
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(self.client.get_blockchain_info().map(|i| i.blocks as u32)?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        let sat_per_kb = self
+            .client
+            .estimate_smart_fee(target as u16, None)?
+            .fee_rate
+            .ok_or(Error::FeeRateUnavailable)?
+            .as_sat() as f64;
+
+        Ok(FeeRate::from_sat_per_vb((sat_per_kb / 1000f64) as f32))
+    }
+}
+
+impl ConfigurableBlockchain for RpcBlockchain {
+    type Config = RpcConfig;
+
+    /// Returns RpcBlockchain backend creating an RPC client to a specific wallet named as the descriptor's checksum
+    /// if it's the first time it creates the wallet in the node and upon return is granted the wallet is loaded
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let wallet_name = config.wallet_name.clone();
+        let wallet_url = format!("{}/wallet/{}", config.url, &wallet_name);
+        debug!("connecting to {} auth:{:?}", wallet_url, config.auth);
+
+        let client = Client::new(wallet_url.as_str(), config.auth.clone().into())?;
+        let loaded_wallets = client.list_wallets()?;
+        if loaded_wallets.contains(&wallet_name) {
+            debug!("wallet already loaded {:?}", wallet_name);
+        } else {
+            let existing_wallets = list_wallet_dir(&client)?;
+            if existing_wallets.contains(&wallet_name) {
+                client.load_wallet(&wallet_name)?;
+                debug!("wallet loaded {:?}", wallet_name);
+            } else {
+                client.create_wallet(&wallet_name, Some(true), None, None, None)?;
+                debug!("wallet created {:?}", wallet_name);
+            }
+        }
+
+        let blockchain_info = client.get_blockchain_info()?;
+        let network = match blockchain_info.chain.as_str() {
+            "main" => Network::Bitcoin,
+            "test" => Network::Testnet,
+            "regtest" => Network::Regtest,
+            "signet" => Network::Signet,
+            _ => return Err(Error::Generic("Invalid network".to_string())),
+        };
+        if network != config.network {
+            return Err(Error::InvalidNetwork {
+                requested: config.network,
+                found: network,
+            });
+        }
+
+        let mut capabilities: HashSet<_> = vec![Capability::FullHistory].into_iter().collect();
+        let rpc_version = client.version()?;
+        if rpc_version >= 210_000 {
+            let info: HashMap<String, Value> = client.call("getindexinfo", &[]).unwrap();
+            if info.contains_key("txindex") {
+                capabilities.insert(Capability::GetAnyTx);
+                capabilities.insert(Capability::AccurateFees);
+            }
+        }
+
+        // this is just a fixed address used only to store a label containing the synced height in the node
+        let mut storage_address =
+            Address::from_str("bc1qst0rewf0wm4kw6qn6kv0e5tc56nkf9yhcxlhqv").unwrap();
+        storage_address.network = network;
+
+        Ok(RpcBlockchain {
+            client,
+            network,
+            capabilities,
+            _storage_address: storage_address,
+            skip_blocks: config.skip_blocks,
+        })
+    }
+}
+
+/// Deterministically generate a unique name given the descriptors defining the wallet
+pub fn wallet_name_from_descriptor<T>(
+    descriptor: T,
+    change_descriptor: Option<T>,
+    network: Network,
+    secp: &SecpCtx,
+) -> Result<String, Error>
+where
+    T: IntoWalletDescriptor,
+{
+    //TODO check descriptors contains only public keys
+    let descriptor = descriptor
+        .into_wallet_descriptor(secp, network)?
+        .0
+        .to_string();
+    let mut wallet_name = get_checksum(&descriptor[..descriptor.find('#').unwrap()])?;
+    if let Some(change_descriptor) = change_descriptor {
+        let change_descriptor = change_descriptor
+            .into_wallet_descriptor(secp, network)?
+            .0
+            .to_string();
+        wallet_name.push_str(
+            get_checksum(&change_descriptor[..change_descriptor.find('#').unwrap()])?.as_str(),
+        );
+    }
+
+    Ok(wallet_name)
+}
+
+/// return the wallets available in default wallet directory
+//TODO use bitcoincore_rpc method when PR #179 lands
+fn list_wallet_dir(client: &Client) -> Result<Vec<String>, Error> {
+    #[derive(Deserialize)]
+    struct Name {
+        name: String,
+    }
+    #[derive(Deserialize)]
+    struct CallResult {
+        wallets: Vec<Name>,
+    }
+
+    let result: CallResult = client.call("listwalletdir", &[])?;
+    Ok(result.wallets.into_iter().map(|n| n.name).collect())
+}
+
+#[cfg(test)]
+#[cfg(feature = "test-rpc")]
+crate::bdk_blockchain_tests! {
+
+    fn test_instance(test_client: &TestClient) -> RpcBlockchain {
+        let config = RpcConfig {
+            url: test_client.bitcoind.rpc_url(),
+            auth: Auth::Cookie { file: test_client.bitcoind.params.cookie_file.clone() },
+            network: Network::Regtest,
+            wallet_name: format!("client-wallet-test-{:?}", std::time::SystemTime::now() ),
+            skip_blocks: None,
+        };
+        RpcBlockchain::from_config(&config).unwrap()
+    }
+}

src/blockchain/utils.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::collections::{HashMap, HashSet};
 
@@ -34,12 +21,12 @@ use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{KeychainKind, TransactionDetails, UTXO};
+use crate::types::{ConfirmationTime, KeychainKind, LocalUtxo, TransactionDetails};
 use crate::wallet::time::Instant;
 use crate::wallet::utils::ChunksIterator;
 
 #[derive(Debug)]
-pub struct ELSGetHistoryRes {
+pub struct ElsGetHistoryRes {
     pub height: i32,
     pub tx_hash: Txid,
 }
@@ -50,7 +37,7 @@ pub trait ElectrumLikeSync {
     fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
         &self,
         scripts: I,
-    ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error>;
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error>;
 
     fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid> + Clone>(
         &self,
@@ -66,7 +53,7 @@ pub trait ElectrumLikeSync {
 
     fn electrum_like_setup<D: BatchDatabase, P: Progress>(
         &self,
-        stop_gap: Option<usize>,
+        stop_gap: usize,
         db: &mut D,
         _progress_update: P,
     ) -> Result<(), Error> {
@@ -74,7 +61,6 @@ pub trait ElectrumLikeSync {
         let start = Instant::new();
         debug!("start setup");
 
-        let stop_gap = stop_gap.unwrap_or(20);
         let chunk_size = stop_gap;
 
         let mut history_txs_id = HashSet::new();
@@ -90,7 +76,7 @@ pub trait ElectrumLikeSync {
 
             for (i, chunk) in ChunksIterator::new(script_iter, stop_gap).enumerate() {
                 // TODO if i == last, should create another chunk of addresses in db
-                let call_result: Vec<Vec<ELSGetHistoryRes>> =
+                let call_result: Vec<Vec<ElsGetHistoryRes>> =
                     maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
                 let max_index = call_result
                     .iter()
@@ -100,7 +86,7 @@ pub trait ElectrumLikeSync {
                 if let Some(max) = max_index {
                     max_indexes.insert(keychain, max + (i * chunk_size) as u32);
                 }
-                let flattened: Vec<ELSGetHistoryRes> = call_result.into_iter().flatten().collect();
+                let flattened: Vec<ElsGetHistoryRes> = call_result.into_iter().flatten().collect();
                 debug!("#{} of {:?} results:{}", i, keychain, flattened.len());
                 if flattened.is_empty() {
                     // Didn't find anything in the last `stop_gap` script_pubkeys, breaking
@@ -160,18 +146,19 @@ pub trait ElectrumLikeSync {
         // save any tx details not in db but in history_txs_id or with different height/timestamp
         for txid in history_txs_id.iter() {
             let height = txid_height.get(txid).cloned().flatten();
-            let timestamp = *new_timestamps.get(txid).unwrap_or(&0u64);
+            let timestamp = new_timestamps.get(txid).cloned();
             if let Some(tx_details) = txs_details_in_db.get(txid) {
-                // check if height matches, otherwise updates it
-                if tx_details.height != height {
+                // check if tx height matches, otherwise updates it. timestamp is not in the if clause
+                // because we are not asking headers for confirmed tx we know about
+                if tx_details.confirmation_time.as_ref().map(|c| c.height) != height {
+                    let confirmation_time = ConfirmationTime::new(height, timestamp);
                     let mut new_tx_details = tx_details.clone();
-                    new_tx_details.height = height;
-                    new_tx_details.timestamp = timestamp;
+                    new_tx_details.confirmation_time = confirmation_time;
                     batch.set_tx(&new_tx_details)?;
                 }
             } else {
                 save_transaction_details_and_utxos(
-                    &txid,
+                    txid,
                     db,
                     timestamp,
                     height,
@@ -184,7 +171,7 @@ pub trait ElectrumLikeSync {
         // remove any tx details in db but not in history_txs_id
         for txid in txs_details_in_db.keys() {
             if !history_txs_id.contains(txid) {
-                batch.del_tx(&txid, false)?;
+                batch.del_tx(txid, false)?;
             }
         }
 
@@ -251,9 +238,13 @@ pub trait ElectrumLikeSync {
         chunk_size: usize,
     ) -> Result<HashMap<Txid, u64>, Error> {
         let mut txid_timestamp = HashMap::new();
+        let txid_in_db_with_conf: HashSet<_> = txs_details_in_db
+            .values()
+            .filter_map(|details| details.confirmation_time.as_ref().map(|_| details.txid))
+            .collect();
         let needed_txid_height: HashMap<&Txid, u32> = txid_height
             .iter()
-            .filter(|(t, _)| txs_details_in_db.get(*t).is_none())
+            .filter(|(t, _)| !txid_in_db_with_conf.contains(*t))
             .filter_map(|(t, o)| o.map(|h| (t, h)))
             .collect();
         let needed_heights: HashSet<u32> = needed_txid_height.values().cloned().collect();
@@ -305,7 +296,7 @@ pub trait ElectrumLikeSync {
 fn save_transaction_details_and_utxos<D: BatchDatabase>(
     txid: &Txid,
     db: &mut D,
-    timestamp: u64,
+    timestamp: Option<u64>,
     height: Option<u32>,
     updates: &mut dyn BatchOperations,
     utxo_deps: &HashMap<OutPoint, OutPoint>,
@@ -342,7 +333,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
 
         // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
         if let Some(outpoint) = utxo_deps.get(&input.previous_output) {
-            updates.del_utxo(&outpoint)?;
+            updates.del_utxo(outpoint)?;
         }
     }
 
@@ -353,7 +344,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
         // this output is ours, we have a path to derive it
         if let Some((keychain, _child)) = db.get_path_from_script_pubkey(&output.script_pubkey)? {
             debug!("{} output #{} is mine, adding utxo", txid, i);
-            updates.set_utxo(&UTXO {
+            updates.set_utxo(&LocalUtxo {
                 outpoint: OutPoint::new(tx.txid(), i as u32),
                 txout: output.clone(),
                 keychain,
@@ -368,9 +359,9 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
         transaction: Some(tx),
         received: incoming,
         sent: outgoing,
-        height,
-        timestamp,
-        fees: inputs_sum.saturating_sub(outputs_sum), /* if the tx is a coinbase, fees would be negative */
+        confirmation_time: ConfirmationTime::new(height, timestamp),
+        fee: Some(inputs_sum.saturating_sub(outputs_sum)), /* if the tx is a coinbase, fees would be negative */
+        verified: height.is_some(),
     };
     updates.set_tx(&tx_details)?;
 
@@ -378,7 +369,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
 }
 
 /// returns utxo dependency as the inputs needed for the utxo to exist
-/// `tx_raw_in_db` must contains utxo's generating txs or errors witt [crate::Error::TransactionNotFound]
+/// `tx_raw_in_db` must contains utxo's generating txs or errors with [crate::Error::TransactionNotFound]
 fn utxos_deps<D: BatchDatabase>(
     db: &mut D,
     tx_raw_in_db: &HashMap<Txid, Transaction>,

src/database/any.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Runtime-checked database types
 //!
@@ -78,6 +65,8 @@ macro_rules! impl_inner_method {
             $enum_name::Memory(inner) => inner.$name( $($args, )* ),
             #[cfg(feature = "key-value-db")]
             $enum_name::Sled(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "sqlite")]
+            $enum_name::Sqlite(inner) => inner.$name( $($args, )* ),
         }
     }
 }
@@ -95,10 +84,15 @@ pub enum AnyDatabase {
     #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
     /// Simple key-value embedded database based on [`sled`]
     Sled(sled::Tree),
+    #[cfg(feature = "sqlite")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))]
+    /// Sqlite embedded database using [`rusqlite`]
+    Sqlite(sqlite::SqliteDatabase),
 }
 
 impl_from!(memory::MemoryDatabase, AnyDatabase, Memory,);
 impl_from!(sled::Tree, AnyDatabase, Sled, #[cfg(feature = "key-value-db")]);
+impl_from!(sqlite::SqliteDatabase, AnyDatabase, Sqlite, #[cfg(feature = "sqlite")]);
 
 /// Type that contains any of the [`BatchDatabase::Batch`] types defined by the library
 pub enum AnyBatch {
@@ -108,6 +102,10 @@ pub enum AnyBatch {
     #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
     /// Simple key-value embedded database based on [`sled`]
     Sled(<sled::Tree as BatchDatabase>::Batch),
+    #[cfg(feature = "sqlite")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))]
+    /// Sqlite embedded database using [`rusqlite`]
+    Sqlite(<sqlite::SqliteDatabase as BatchDatabase>::Batch),
 }
 
 impl_from!(
@@ -116,6 +114,7 @@ impl_from!(
     Memory,
 );
 impl_from!(<sled::Tree as BatchDatabase>::Batch, AnyBatch, Sled, #[cfg(feature = "key-value-db")]);
+impl_from!(<sqlite::SqliteDatabase as BatchDatabase>::Batch, AnyBatch, Sqlite, #[cfg(feature = "sqlite")]);
 
 impl BatchOperations for AnyDatabase {
     fn set_script_pubkey(
@@ -133,7 +132,7 @@ impl BatchOperations for AnyDatabase {
             child
         )
     }
-    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
         impl_inner_method!(AnyDatabase, self, set_utxo, utxo)
     }
     fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
@@ -165,7 +164,7 @@ impl BatchOperations for AnyDatabase {
     ) -> Result<Option<(KeychainKind, u32)>, Error> {
         impl_inner_method!(AnyDatabase, self, del_path_from_script_pubkey, script)
     }
-    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
         impl_inner_method!(AnyDatabase, self, del_utxo, outpoint)
     }
     fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
@@ -201,7 +200,7 @@ impl Database for AnyDatabase {
     fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
         impl_inner_method!(AnyDatabase, self, iter_script_pubkeys, keychain)
     }
-    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error> {
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
         impl_inner_method!(AnyDatabase, self, iter_utxos)
     }
     fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error> {
@@ -230,7 +229,7 @@ impl Database for AnyDatabase {
     ) -> Result<Option<(KeychainKind, u32)>, Error> {
         impl_inner_method!(AnyDatabase, self, get_path_from_script_pubkey, script)
     }
-    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
         impl_inner_method!(AnyDatabase, self, get_utxo, outpoint)
     }
     fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
@@ -246,6 +245,10 @@ impl Database for AnyDatabase {
     fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
         impl_inner_method!(AnyDatabase, self, increment_last_index, keychain)
     }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, flush)
+    }
 }
 
 impl BatchOperations for AnyBatch {
@@ -257,7 +260,7 @@ impl BatchOperations for AnyBatch {
     ) -> Result<(), Error> {
         impl_inner_method!(AnyBatch, self, set_script_pubkey, script, keychain, child)
     }
-    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
         impl_inner_method!(AnyBatch, self, set_utxo, utxo)
     }
     fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
@@ -283,7 +286,7 @@ impl BatchOperations for AnyBatch {
     ) -> Result<Option<(KeychainKind, u32)>, Error> {
         impl_inner_method!(AnyBatch, self, del_path_from_script_pubkey, script)
     }
-    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
         impl_inner_method!(AnyBatch, self, del_utxo, outpoint)
     }
     fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
@@ -309,27 +312,27 @@ impl BatchDatabase for AnyDatabase {
             AnyDatabase::Memory(inner) => inner.begin_batch().into(),
             #[cfg(feature = "key-value-db")]
             AnyDatabase::Sled(inner) => inner.begin_batch().into(),
+            #[cfg(feature = "sqlite")]
+            AnyDatabase::Sqlite(inner) => inner.begin_batch().into(),
         }
     }
     fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> {
-        // TODO: refactor once `move_ref_pattern` is stable
-        #[allow(irrefutable_let_patterns)]
         match self {
-            AnyDatabase::Memory(db) => {
-                if let AnyBatch::Memory(batch) = batch {
-                    db.commit_batch(batch)
-                } else {
-                    unimplemented!()
-                }
-            }
+            AnyDatabase::Memory(db) => match batch {
+                AnyBatch::Memory(batch) => db.commit_batch(batch),
+                #[cfg(any(feature = "key-value-db", feature = "sqlite"))]
+                _ => unimplemented!("Other batch shouldn't be used with Memory db."),
+            },
             #[cfg(feature = "key-value-db")]
-            AnyDatabase::Sled(db) => {
-                if let AnyBatch::Sled(batch) = batch {
-                    db.commit_batch(batch)
-                } else {
-                    unimplemented!()
-                }
-            }
+            AnyDatabase::Sled(db) => match batch {
+                AnyBatch::Sled(batch) => db.commit_batch(batch),
+                _ => unimplemented!("Other batch shouldn't be used with Sled db."),
+            },
+            #[cfg(feature = "sqlite")]
+            AnyDatabase::Sqlite(db) => match batch {
+                AnyBatch::Sqlite(batch) => db.commit_batch(batch),
+                _ => unimplemented!("Other batch shouldn't be used with Sqlite db."),
+            },
         }
     }
 }
@@ -353,6 +356,23 @@ impl ConfigurableDatabase for sled::Tree {
     }
 }
 
+/// Configuration type for a [`sqlite::SqliteDatabase`] database
+#[cfg(feature = "sqlite")]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct SqliteDbConfiguration {
+    /// Main directory of the db
+    pub path: String,
+}
+
+#[cfg(feature = "sqlite")]
+impl ConfigurableDatabase for sqlite::SqliteDatabase {
+    type Config = SqliteDbConfiguration;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(sqlite::SqliteDatabase::new(config.path.clone()))
+    }
+}
+
 /// Type that can contain any of the database configurations defined by the library
 ///
 /// This allows storing a single configuration that can be loaded into an [`AnyDatabase`]
@@ -366,6 +386,10 @@ pub enum AnyDatabaseConfig {
     #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
     /// Simple key-value embedded database based on [`sled`]
     Sled(SledDbConfiguration),
+    #[cfg(feature = "sqlite")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))]
+    /// Sqlite embedded database using [`rusqlite`]
+    Sqlite(SqliteDbConfiguration),
 }
 
 impl ConfigurableDatabase for AnyDatabase {
@@ -378,9 +402,14 @@ impl ConfigurableDatabase for AnyDatabase {
             }
             #[cfg(feature = "key-value-db")]
             AnyDatabaseConfig::Sled(inner) => AnyDatabase::Sled(sled::Tree::from_config(inner)?),
+            #[cfg(feature = "sqlite")]
+            AnyDatabaseConfig::Sqlite(inner) => {
+                AnyDatabase::Sqlite(sqlite::SqliteDatabase::from_config(inner)?)
+            }
         })
     }
 }
 
 impl_from!((), AnyDatabaseConfig, Memory,);
 impl_from!(SledDbConfiguration, AnyDatabaseConfig, Sled, #[cfg(feature = "key-value-db")]);
+impl_from!(SqliteDbConfiguration, AnyDatabaseConfig, Sqlite, #[cfg(feature = "sqlite")]);

src/database/keyvalue.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::convert::TryInto;
 
@@ -51,8 +38,8 @@ macro_rules! impl_batch_operations {
             Ok(())
         }
 
-        fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
-            let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
+        fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+            let key = MapKey::Utxo(Some(&utxo.outpoint)).as_map_key();
             let value = json!({
                 "t": utxo.txout,
                 "i": utxo.keychain,
@@ -120,8 +107,8 @@ macro_rules! impl_batch_operations {
             }
         }
 
-        fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-            let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+        fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+            let key = MapKey::Utxo(Some(outpoint)).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
@@ -132,7 +119,7 @@ macro_rules! impl_batch_operations {
                     let txout = serde_json::from_value(val["t"].take())?;
                     let keychain = serde_json::from_value(val["i"].take())?;
 
-                    Ok(Some(UTXO { outpoint: outpoint.clone(), txout, keychain }))
+                    Ok(Some(LocalUtxo { outpoint: outpoint.clone(), txout, keychain }))
                 }
             }
         }
@@ -234,8 +221,8 @@ impl Database for Tree {
             .collect()
     }
 
-    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error> {
-        let key = MapKey::UTXO(None).as_map_key();
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(None).as_map_key();
         self.scan_prefix(key)
             .map(|x| -> Result<_, Error> {
                 let (k, v) = x?;
@@ -245,7 +232,7 @@ impl Database for Tree {
                 let txout = serde_json::from_value(val["t"].take())?;
                 let keychain = serde_json::from_value(val["i"].take())?;
 
-                Ok(UTXO {
+                Ok(LocalUtxo {
                     outpoint,
                     txout,
                     keychain,
@@ -305,15 +292,15 @@ impl Database for Tree {
             .transpose()
     }
 
-    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-        let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(Some(outpoint)).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
                 let mut val: serde_json::Value = serde_json::from_slice(&b)?;
                 let txout = serde_json::from_value(val["t"].take())?;
                 let keychain = serde_json::from_value(val["i"].take())?;
 
-                Ok(UTXO {
+                Ok(LocalUtxo {
                     outpoint: *outpoint,
                     txout,
                     keychain,
@@ -333,7 +320,7 @@ impl Database for Tree {
             .map(|b| -> Result<_, Error> {
                 let mut txdetails: TransactionDetails = serde_json::from_slice(&b)?;
                 if include_raw {
-                    txdetails.transaction = self.get_raw_tx(&txid)?;
+                    txdetails.transaction = self.get_raw_tx(txid)?;
                 }
 
                 Ok(txdetails)
@@ -380,6 +367,10 @@ impl Database for Tree {
             Ok(val)
         })
     }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        Ok(Tree::flush(self).map(|_| ())?)
+    }
 }
 
 impl BatchDatabase for Tree {
@@ -396,6 +387,7 @@ impl BatchDatabase for Tree {
 
 #[cfg(test)]
 mod test {
+    use lazy_static::lazy_static;
     use std::sync::{Arc, Condvar, Mutex, Once};
     use std::time::{SystemTime, UNIX_EPOCH};
 

src/database/memory.rs

@@ -1,32 +1,20 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! In-memory ephemeral database
 //!
 //! This module defines an in-memory database type called [`MemoryDatabase`] that is based on a
 //! [`BTreeMap`].
 
+use std::any::Any;
 use std::collections::BTreeMap;
 use std::ops::Bound::{Excluded, Included};
 
@@ -49,7 +37,7 @@ use crate::types::*;
 pub(crate) enum MapKey<'a> {
     Path((Option<KeychainKind>, Option<u32>)),
     Script(Option<&'a Script>),
-    UTXO(Option<&'a OutPoint>),
+    Utxo(Option<&'a OutPoint>),
     RawTx(Option<&'a Txid>),
     Transaction(Option<&'a Txid>),
     LastIndex(KeychainKind),
@@ -67,7 +55,7 @@ impl MapKey<'_> {
                 v
             }
             MapKey::Script(_) => b"s".to_vec(),
-            MapKey::UTXO(_) => b"u".to_vec(),
+            MapKey::Utxo(_) => b"u".to_vec(),
             MapKey::RawTx(_) => b"r".to_vec(),
             MapKey::Transaction(_) => b"t".to_vec(),
             MapKey::LastIndex(st) => [b"c", st.as_ref()].concat(),
@@ -79,7 +67,7 @@ impl MapKey<'_> {
         match self {
             MapKey::Path((_, Some(child))) => child.to_be_bytes().to_vec(),
             MapKey::Script(Some(s)) => serialize(*s),
-            MapKey::UTXO(Some(s)) => serialize(*s),
+            MapKey::Utxo(Some(s)) => serialize(*s),
             MapKey::RawTx(Some(s)) => serialize(*s),
             MapKey::Transaction(Some(s)) => serialize(*s),
             _ => vec![],
@@ -123,7 +111,7 @@ fn after(key: &[u8]) -> Vec<u8> {
 /// [`database`]: crate::database
 #[derive(Debug, Default)]
 pub struct MemoryDatabase {
-    map: BTreeMap<Vec<u8>, Box<dyn std::any::Any>>,
+    map: BTreeMap<Vec<u8>, Box<dyn Any + Send + Sync>>,
     deleted_keys: Vec<Vec<u8>>,
 }
 
@@ -157,8 +145,8 @@ impl BatchOperations for MemoryDatabase {
         Ok(())
     }
 
-    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
-        let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+        let key = MapKey::Utxo(Some(&utxo.outpoint)).as_map_key();
         self.map
             .insert(key, Box::new((utxo.txout.clone(), utxo.keychain)));
 
@@ -223,8 +211,8 @@ impl BatchOperations for MemoryDatabase {
             }
         }
     }
-    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-        let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(Some(outpoint)).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
@@ -232,7 +220,7 @@ impl BatchOperations for MemoryDatabase {
             None => Ok(None),
             Some(b) => {
                 let (txout, keychain) = b.downcast_ref().cloned().unwrap();
-                Ok(Some(UTXO {
+                Ok(Some(LocalUtxo {
                     outpoint: *outpoint,
                     txout,
                     keychain,
@@ -316,14 +304,14 @@ impl Database for MemoryDatabase {
             .collect()
     }
 
-    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error> {
-        let key = MapKey::UTXO(None).as_map_key();
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(None).as_map_key();
         self.map
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(k, v)| {
                 let outpoint = deserialize(&k[1..]).unwrap();
                 let (txout, keychain) = v.downcast_ref().cloned().unwrap();
-                Ok(UTXO {
+                Ok(LocalUtxo {
                     outpoint,
                     txout,
                     keychain,
@@ -382,11 +370,11 @@ impl Database for MemoryDatabase {
         }))
     }
 
-    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-        let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(Some(outpoint)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
             let (txout, keychain) = b.downcast_ref().cloned().unwrap();
-            UTXO {
+            LocalUtxo {
                 outpoint: *outpoint,
                 txout,
                 keychain,
@@ -407,7 +395,7 @@ impl Database for MemoryDatabase {
         Ok(self.map.get(&key).map(|b| {
             let mut txdetails: TransactionDetails = b.downcast_ref().cloned().unwrap();
             if include_raw {
-                txdetails.transaction = self.get_raw_tx(&txid).unwrap();
+                txdetails.transaction = self.get_raw_tx(txid).unwrap();
             }
 
             txdetails
@@ -432,6 +420,10 @@ impl Database for MemoryDatabase {
 
         Ok(*value)
     }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        Ok(())
+    }
 }
 
 impl BatchDatabase for MemoryDatabase {
@@ -442,8 +434,8 @@ impl BatchDatabase for MemoryDatabase {
     }
 
     fn commit_batch(&mut self, mut batch: Self::Batch) -> Result<(), Error> {
-        for key in batch.deleted_keys {
-            self.map.remove(&key);
+        for key in batch.deleted_keys.iter() {
+            self.map.remove(key);
         }
         self.map.append(&mut batch.map);
         Ok(())
@@ -465,20 +457,21 @@ impl ConfigurableDatabase for MemoryDatabase {
 /// don't have `test` set.
 macro_rules! populate_test_db {
     ($db:expr, $tx_meta:expr, $current_height:expr$(,)?) => {{
+        use std::str::FromStr;
         use $crate::database::BatchOperations;
         let mut db = $db;
         let tx_meta = $tx_meta;
         let current_height: Option<u32> = $current_height;
-        let tx = Transaction {
+        let tx = $crate::bitcoin::Transaction {
             version: 1,
             lock_time: 0,
             input: vec![],
             output: tx_meta
                 .output
                 .iter()
-                .map(|out_meta| bitcoin::TxOut {
+                .map(|out_meta| $crate::bitcoin::TxOut {
                     value: out_meta.value,
-                    script_pubkey: bitcoin::Address::from_str(&out_meta.to_address)
+                    script_pubkey: $crate::bitcoin::Address::from_str(&out_meta.to_address)
                         .unwrap()
                         .script_pubkey(),
                 })
@@ -486,29 +479,32 @@ macro_rules! populate_test_db {
         };
 
         let txid = tx.txid();
-        let height = tx_meta
+        let confirmation_time = tx_meta
             .min_confirmations
-            .map(|conf| current_height.unwrap().checked_sub(conf as u32).unwrap());
+            .map(|conf| $crate::ConfirmationTime {
+                height: current_height.unwrap().checked_sub(conf as u32).unwrap(),
+                timestamp: 0,
+            });
 
-        let tx_details = TransactionDetails {
+        let tx_details = $crate::TransactionDetails {
             transaction: Some(tx.clone()),
             txid,
-            timestamp: 0,
-            height,
+            fee: Some(0),
             received: 0,
             sent: 0,
-            fees: 0,
+            confirmation_time,
+            verified: current_height.is_some(),
         };
 
         db.set_tx(&tx_details).unwrap();
         for (vout, out) in tx.output.iter().enumerate() {
-            db.set_utxo(&UTXO {
+            db.set_utxo(&$crate::LocalUtxo {
                 txout: out.clone(),
-                outpoint: OutPoint {
+                outpoint: $crate::bitcoin::OutPoint {
                     txid,
                     vout: vout as u32,
                 },
-                keychain: KeychainKind::External,
+                keychain: $crate::KeychainKind::External,
             })
             .unwrap();
         }
@@ -524,7 +520,7 @@ macro_rules! doctest_wallet {
     () => {{
         use $crate::bitcoin::Network;
         use $crate::database::MemoryDatabase;
-        use testutils::testutils;
+        use $crate::testutils;
         let descriptor = "wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)";
         let descriptors = testutils!(@descriptors (descriptor) (descriptor));
 

src/database/mod.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Database types
 //!
@@ -49,6 +36,11 @@ pub use any::{AnyDatabase, AnyDatabaseConfig};
 #[cfg(feature = "key-value-db")]
 pub(crate) mod keyvalue;
 
+#[cfg(feature = "sqlite")]
+pub(crate) mod sqlite;
+#[cfg(feature = "sqlite")]
+pub use sqlite::SqliteDatabase;
+
 pub mod memory;
 pub use memory::MemoryDatabase;
 
@@ -64,8 +56,8 @@ pub trait BatchOperations {
         keychain: KeychainKind,
         child: u32,
     ) -> Result<(), Error>;
-    /// Store a [`UTXO`]
-    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error>;
+    /// Store a [`LocalUtxo`]
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error>;
     /// Store a raw transaction
     fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error>;
     /// Store the metadata of a transaction
@@ -85,8 +77,8 @@ pub trait BatchOperations {
         &mut self,
         script: &Script,
     ) -> Result<Option<(KeychainKind, u32)>, Error>;
-    /// Delete a [`UTXO`] given its [`OutPoint`]
-    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
+    /// Delete a [`LocalUtxo`] given its [`OutPoint`]
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error>;
     /// Delete a raw transaction given its [`Txid`]
     fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error>;
     /// Delete the metadata of a transaction and optionally the raw transaction itself
@@ -116,8 +108,8 @@ pub trait Database: BatchOperations {
 
     /// Return the list of script_pubkeys
     fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error>;
-    /// Return the list of [`UTXO`]s
-    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error>;
+    /// Return the list of [`LocalUtxo`]s
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error>;
     /// Return the list of raw transactions
     fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error>;
     /// Return the list of transactions metadata
@@ -134,8 +126,8 @@ pub trait Database: BatchOperations {
         &self,
         script: &Script,
     ) -> Result<Option<(KeychainKind, u32)>, Error>;
-    /// Fetch a [`UTXO`] given its [`OutPoint`]
-    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
+    /// Fetch a [`LocalUtxo`] given its [`OutPoint`]
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error>;
     /// Fetch a raw transaction given its [`Txid`]
     fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error>;
     /// Fetch the transaction metadata and optionally also the raw transaction
@@ -147,6 +139,9 @@ pub trait Database: BatchOperations {
     ///
     /// It should insert and return `0` if not present in the database
     fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error>;
+
+    /// Force changes to be written to disk
+    fn flush(&mut self) -> Result<(), Error>;
 }
 
 /// Trait for a database that supports batch operations
@@ -177,14 +172,14 @@ pub(crate) trait DatabaseUtils: Database {
             .map(|o| o.is_some())
     }
 
-    fn get_raw_tx_or<F>(&self, txid: &Txid, f: F) -> Result<Option<Transaction>, Error>
+    fn get_raw_tx_or<D>(&self, txid: &Txid, default: D) -> Result<Option<Transaction>, Error>
     where
-        F: FnOnce() -> Result<Option<Transaction>, Error>,
+        D: FnOnce() -> Result<Option<Transaction>, Error>,
     {
         self.get_tx(txid, true)?
             .map(|t| t.transaction)
             .flatten()
-            .map_or_else(f, |t| Ok(Some(t)))
+            .map_or_else(default, |t| Ok(Some(t)))
     }
 
     fn get_previous_output(&self, outpoint: &OutPoint) -> Result<Option<TxOut>, Error> {
@@ -227,7 +222,7 @@ pub mod test {
         );
         assert_eq!(
             tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((keychain, path.clone()))
+            Some((keychain, path))
         );
     }
 
@@ -256,7 +251,7 @@ pub mod test {
         );
         assert_eq!(
             tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((keychain, path.clone()))
+            Some((keychain, path))
         );
     }
 
@@ -298,7 +293,7 @@ pub mod test {
             value: 133742,
             script_pubkey: script,
         };
-        let utxo = UTXO {
+        let utxo = LocalUtxo {
             txout,
             outpoint,
             keychain: KeychainKind::External,
@@ -327,11 +322,14 @@ pub mod test {
         let mut tx_details = TransactionDetails {
             transaction: Some(tx),
             txid,
-            timestamp: 123456,
             received: 1337,
             sent: 420420,
-            fees: 140,
-            height: Some(1000),
+            fee: Some(140),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 123456,
+                height: 1000,
+            }),
+            verified: true,
         };
 
         tree.set_tx(&tx_details).unwrap();

src/database/sqlite.rs

@@ -0,0 +1,968 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::hash_types::Txid;
+use bitcoin::{OutPoint, Script, Transaction, TxOut};
+
+use crate::database::{BatchDatabase, BatchOperations, Database};
+use crate::error::Error;
+use crate::types::*;
+
+use rusqlite::{named_params, Connection};
+
+static MIGRATIONS: &[&str] = &[
+    "CREATE TABLE version (version INTEGER)",
+    "INSERT INTO version VALUES (1)",
+    "CREATE TABLE script_pubkeys (keychain TEXT, child INTEGER, script BLOB);",
+    "CREATE INDEX idx_keychain_child ON script_pubkeys(keychain, child);",
+    "CREATE INDEX idx_script ON script_pubkeys(script);",
+    "CREATE TABLE utxos (value INTEGER, keychain TEXT, vout INTEGER, txid BLOB, script BLOB);",
+    "CREATE INDEX idx_txid_vout ON utxos(txid, vout);",
+    "CREATE TABLE transactions (txid BLOB, raw_tx BLOB);",
+    "CREATE INDEX idx_txid ON transactions(txid);",
+    "CREATE TABLE transaction_details (txid BLOB, timestamp INTEGER, received INTEGER, sent INTEGER, fee INTEGER, height INTEGER, verified INTEGER DEFAULT 0);",
+    "CREATE INDEX idx_txdetails_txid ON transaction_details(txid);",
+    "CREATE TABLE last_derivation_indices (keychain TEXT, value INTEGER);",
+    "CREATE UNIQUE INDEX idx_indices_keychain ON last_derivation_indices(keychain);",
+    "CREATE TABLE checksums (keychain TEXT, checksum BLOB);",
+    "CREATE INDEX idx_checksums_keychain ON checksums(keychain);",
+];
+
+/// Sqlite database stored on filesystem
+///
+/// This is a permanent storage solution for devices and platforms that provide a filesystem.
+/// [`crate::database`]
+#[derive(Debug)]
+pub struct SqliteDatabase {
+    /// Path on the local filesystem to store the sqlite file
+    pub path: String,
+    /// A rusqlite connection object to the sqlite database
+    pub connection: Connection,
+}
+
+impl SqliteDatabase {
+    /// Instantiate a new SqliteDatabase instance by creating a connection
+    /// to the database stored at path
+    pub fn new(path: String) -> Self {
+        let connection = get_connection(&path).unwrap();
+        SqliteDatabase { path, connection }
+    }
+    fn insert_script_pubkey(
+        &self,
+        keychain: String,
+        child: u32,
+        script: &[u8],
+    ) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached("INSERT INTO script_pubkeys (keychain, child, script) VALUES (:keychain, :child, :script)")?;
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":child": child,
+            ":script": script
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+    fn insert_utxo(
+        &self,
+        value: u64,
+        keychain: String,
+        vout: u32,
+        txid: &[u8],
+        script: &[u8],
+    ) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached("INSERT INTO utxos (value, keychain, vout, txid, script) VALUES (:value, :keychain, :vout, :txid, :script)")?;
+        statement.execute(named_params! {
+            ":value": value,
+            ":keychain": keychain,
+            ":vout": vout,
+            ":txid": txid,
+            ":script": script
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+    fn insert_transaction(&self, txid: &[u8], raw_tx: &[u8]) -> Result<i64, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("INSERT INTO transactions (txid, raw_tx) VALUES (:txid, :raw_tx)")?;
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":raw_tx": raw_tx,
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn update_transaction(&self, txid: &[u8], raw_tx: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("UPDATE transactions SET raw_tx=:raw_tx WHERE txid=:txid")?;
+
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":raw_tx": raw_tx,
+        })?;
+
+        Ok(())
+    }
+
+    fn insert_transaction_details(&self, transaction: &TransactionDetails) -> Result<i64, Error> {
+        let (timestamp, height) = match &transaction.confirmation_time {
+            Some(confirmation_time) => (
+                Some(confirmation_time.timestamp),
+                Some(confirmation_time.height),
+            ),
+            None => (None, None),
+        };
+
+        let txid: &[u8] = &transaction.txid;
+
+        let mut statement = self.connection.prepare_cached("INSERT INTO transaction_details (txid, timestamp, received, sent, fee, height, verified) VALUES (:txid, :timestamp, :received, :sent, :fee, :height, :verified)")?;
+
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":timestamp": timestamp,
+            ":received": transaction.received,
+            ":sent": transaction.sent,
+            ":fee": transaction.fee,
+            ":height": height,
+            ":verified": transaction.verified
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn update_transaction_details(&self, transaction: &TransactionDetails) -> Result<(), Error> {
+        let (timestamp, height) = match &transaction.confirmation_time {
+            Some(confirmation_time) => (
+                Some(confirmation_time.timestamp),
+                Some(confirmation_time.height),
+            ),
+            None => (None, None),
+        };
+
+        let txid: &[u8] = &transaction.txid;
+
+        let mut statement = self.connection.prepare_cached("UPDATE transaction_details SET timestamp=:timestamp, received=:received, sent=:sent, fee=:fee, height=:height, verified=:verified WHERE txid=:txid")?;
+
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":timestamp": timestamp,
+            ":received": transaction.received,
+            ":sent": transaction.sent,
+            ":fee": transaction.fee,
+            ":height": height,
+            ":verified": transaction.verified,
+        })?;
+
+        Ok(())
+    }
+
+    fn insert_last_derivation_index(&self, keychain: String, value: u32) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "INSERT INTO last_derivation_indices (keychain, value) VALUES (:keychain, :value)",
+        )?;
+
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":value": value,
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn insert_checksum(&self, keychain: String, checksum: &[u8]) -> Result<i64, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "INSERT INTO checksums (keychain, checksum) VALUES (:keychain, :checksum)",
+        )?;
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":checksum": checksum,
+        })?;
+
+        Ok(self.connection.last_insert_rowid())
+    }
+
+    fn update_last_derivation_index(&self, keychain: String, value: u32) -> Result<(), Error> {
+        let mut statement = self.connection.prepare_cached(
+            "INSERT INTO last_derivation_indices (keychain, value) VALUES (:keychain, :value) ON CONFLICT(keychain) DO UPDATE SET value=:value WHERE keychain=:keychain",
+        )?;
+
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":value": value,
+        })?;
+
+        Ok(())
+    }
+
+    fn select_script_pubkeys(&self) -> Result<Vec<Script>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT script FROM script_pubkeys")?;
+        let mut scripts: Vec<Script> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let raw_script: Vec<u8> = row.get(0)?;
+            scripts.push(raw_script.into());
+        }
+
+        Ok(scripts)
+    }
+
+    fn select_script_pubkeys_by_keychain(&self, keychain: String) -> Result<Vec<Script>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT script FROM script_pubkeys WHERE keychain=:keychain")?;
+        let mut scripts: Vec<Script> = vec![];
+        let mut rows = statement.query(named_params! {":keychain": keychain})?;
+        while let Some(row) = rows.next()? {
+            let raw_script: Vec<u8> = row.get(0)?;
+            scripts.push(raw_script.into());
+        }
+
+        Ok(scripts)
+    }
+
+    fn select_script_pubkey_by_path(
+        &self,
+        keychain: String,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "SELECT script FROM script_pubkeys WHERE keychain=:keychain AND child=:child",
+        )?;
+        let mut rows = statement.query(named_params! {":keychain": keychain,":child": child})?;
+
+        match rows.next()? {
+            Some(row) => {
+                let script: Vec<u8> = row.get(0)?;
+                let script: Script = script.into();
+                Ok(Some(script))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_script_pubkey_by_script(
+        &self,
+        script: &[u8],
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT keychain, child FROM script_pubkeys WHERE script=:script")?;
+        let mut rows = statement.query(named_params! {":script": script})?;
+        match rows.next()? {
+            Some(row) => {
+                let keychain: String = row.get(0)?;
+                let keychain: KeychainKind = serde_json::from_str(&keychain)?;
+                let child: u32 = row.get(1)?;
+                Ok(Some((keychain, child)))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT value, keychain, vout, txid, script FROM utxos")?;
+        let mut utxos: Vec<LocalUtxo> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let value = row.get(0)?;
+            let keychain: String = row.get(1)?;
+            let vout = row.get(2)?;
+            let txid: Vec<u8> = row.get(3)?;
+            let script: Vec<u8> = row.get(4)?;
+
+            let keychain: KeychainKind = serde_json::from_str(&keychain)?;
+
+            utxos.push(LocalUtxo {
+                outpoint: OutPoint::new(deserialize(&txid)?, vout),
+                txout: TxOut {
+                    value,
+                    script_pubkey: script.into(),
+                },
+                keychain,
+            })
+        }
+
+        Ok(utxos)
+    }
+
+    fn select_utxo_by_outpoint(
+        &self,
+        txid: &[u8],
+        vout: u32,
+    ) -> Result<Option<(u64, KeychainKind, Script)>, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "SELECT value, keychain, script FROM utxos WHERE txid=:txid AND vout=:vout",
+        )?;
+        let mut rows = statement.query(named_params! {":txid": txid,":vout": vout})?;
+        match rows.next()? {
+            Some(row) => {
+                let value: u64 = row.get(0)?;
+                let keychain: String = row.get(1)?;
+                let keychain: KeychainKind = serde_json::from_str(&keychain)?;
+                let script: Vec<u8> = row.get(2)?;
+                let script: Script = script.into();
+
+                Ok(Some((value, keychain, script)))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_transactions(&self) -> Result<Vec<Transaction>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT raw_tx FROM transactions")?;
+        let mut txs: Vec<Transaction> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let raw_tx: Vec<u8> = row.get(0)?;
+            let tx: Transaction = deserialize(&raw_tx)?;
+            txs.push(tx);
+        }
+        Ok(txs)
+    }
+
+    fn select_transaction_by_txid(&self, txid: &[u8]) -> Result<Option<Transaction>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT raw_tx FROM transactions WHERE txid=:txid")?;
+        let mut rows = statement.query(named_params! {":txid": txid})?;
+        match rows.next()? {
+            Some(row) => {
+                let raw_tx: Vec<u8> = row.get(0)?;
+                let tx: Transaction = deserialize(&raw_tx)?;
+                Ok(Some(tx))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_transaction_details_with_raw(&self) -> Result<Vec<TransactionDetails>, Error> {
+        let mut statement = self.connection.prepare_cached("SELECT transaction_details.txid, transaction_details.timestamp, transaction_details.received, transaction_details.sent, transaction_details.fee, transaction_details.height, transaction_details.verified, transactions.raw_tx FROM transaction_details, transactions WHERE transaction_details.txid = transactions.txid")?;
+        let mut transaction_details: Vec<TransactionDetails> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let txid: Vec<u8> = row.get(0)?;
+            let txid: Txid = deserialize(&txid)?;
+            let timestamp: Option<u64> = row.get(1)?;
+            let received: u64 = row.get(2)?;
+            let sent: u64 = row.get(3)?;
+            let fee: Option<u64> = row.get(4)?;
+            let height: Option<u32> = row.get(5)?;
+            let verified: bool = row.get(6)?;
+            let raw_tx: Option<Vec<u8>> = row.get(7)?;
+            let tx: Option<Transaction> = match raw_tx {
+                Some(raw_tx) => {
+                    let tx: Transaction = deserialize(&raw_tx)?;
+                    Some(tx)
+                }
+                None => None,
+            };
+
+            let confirmation_time = match (height, timestamp) {
+                (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+                _ => None,
+            };
+
+            transaction_details.push(TransactionDetails {
+                transaction: tx,
+                txid,
+                received,
+                sent,
+                fee,
+                confirmation_time,
+                verified,
+            });
+        }
+        Ok(transaction_details)
+    }
+
+    fn select_transaction_details(&self) -> Result<Vec<TransactionDetails>, Error> {
+        let mut statement = self.connection.prepare_cached(
+            "SELECT txid, timestamp, received, sent, fee, height, verified FROM transaction_details",
+        )?;
+        let mut transaction_details: Vec<TransactionDetails> = vec![];
+        let mut rows = statement.query([])?;
+        while let Some(row) = rows.next()? {
+            let txid: Vec<u8> = row.get(0)?;
+            let txid: Txid = deserialize(&txid)?;
+            let timestamp: Option<u64> = row.get(1)?;
+            let received: u64 = row.get(2)?;
+            let sent: u64 = row.get(3)?;
+            let fee: Option<u64> = row.get(4)?;
+            let height: Option<u32> = row.get(5)?;
+            let verified: bool = row.get(6)?;
+
+            let confirmation_time = match (height, timestamp) {
+                (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+                _ => None,
+            };
+
+            transaction_details.push(TransactionDetails {
+                transaction: None,
+                txid,
+                received,
+                sent,
+                fee,
+                confirmation_time,
+                verified,
+            });
+        }
+        Ok(transaction_details)
+    }
+
+    fn select_transaction_details_by_txid(
+        &self,
+        txid: &[u8],
+    ) -> Result<Option<TransactionDetails>, Error> {
+        let mut statement = self.connection.prepare_cached("SELECT transaction_details.timestamp, transaction_details.received, transaction_details.sent, transaction_details.fee, transaction_details.height, transaction_details.verified, transactions.raw_tx FROM transaction_details, transactions WHERE transaction_details.txid=transactions.txid AND transaction_details.txid=:txid")?;
+        let mut rows = statement.query(named_params! { ":txid": txid })?;
+
+        match rows.next()? {
+            Some(row) => {
+                let timestamp: Option<u64> = row.get(0)?;
+                let received: u64 = row.get(1)?;
+                let sent: u64 = row.get(2)?;
+                let fee: Option<u64> = row.get(3)?;
+                let height: Option<u32> = row.get(4)?;
+                let verified: bool = row.get(5)?;
+
+                let raw_tx: Option<Vec<u8>> = row.get(6)?;
+                let tx: Option<Transaction> = match raw_tx {
+                    Some(raw_tx) => {
+                        let tx: Transaction = deserialize(&raw_tx)?;
+                        Some(tx)
+                    }
+                    None => None,
+                };
+
+                let confirmation_time = match (height, timestamp) {
+                    (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+                    _ => None,
+                };
+
+                Ok(Some(TransactionDetails {
+                    transaction: tx,
+                    txid: deserialize(txid)?,
+                    received,
+                    sent,
+                    fee,
+                    confirmation_time,
+                    verified,
+                }))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_last_derivation_index_by_keychain(
+        &self,
+        keychain: String,
+    ) -> Result<Option<u32>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT value FROM last_derivation_indices WHERE keychain=:keychain")?;
+        let mut rows = statement.query(named_params! {":keychain": keychain})?;
+        match rows.next()? {
+            Some(row) => {
+                let value: u32 = row.get(0)?;
+                Ok(Some(value))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn select_checksum_by_keychain(&self, keychain: String) -> Result<Option<Vec<u8>>, Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("SELECT checksum FROM checksums WHERE keychain=:keychain")?;
+        let mut rows = statement.query(named_params! {":keychain": keychain})?;
+
+        match rows.next()? {
+            Some(row) => {
+                let checksum: Vec<u8> = row.get(0)?;
+                Ok(Some(checksum))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn delete_script_pubkey_by_path(&self, keychain: String, child: u32) -> Result<(), Error> {
+        let mut statement = self.connection.prepare_cached(
+            "DELETE FROM script_pubkeys WHERE keychain=:keychain AND child=:child",
+        )?;
+        statement.execute(named_params! {
+            ":keychain": keychain,
+            ":child": child
+        })?;
+
+        Ok(())
+    }
+
+    fn delete_script_pubkey_by_script(&self, script: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM script_pubkeys WHERE script=:script")?;
+        statement.execute(named_params! {
+            ":script": script
+        })?;
+
+        Ok(())
+    }
+
+    fn delete_utxo_by_outpoint(&self, txid: &[u8], vout: u32) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM utxos WHERE txid=:txid AND vout=:vout")?;
+        statement.execute(named_params! {
+            ":txid": txid,
+            ":vout": vout
+        })?;
+
+        Ok(())
+    }
+
+    fn delete_transaction_by_txid(&self, txid: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM transactions WHERE txid=:txid")?;
+        statement.execute(named_params! {":txid": txid})?;
+        Ok(())
+    }
+
+    fn delete_transaction_details_by_txid(&self, txid: &[u8]) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM transaction_details WHERE txid=:txid")?;
+        statement.execute(named_params! {":txid": txid})?;
+        Ok(())
+    }
+
+    fn delete_last_derivation_index_by_keychain(&self, keychain: String) -> Result<(), Error> {
+        let mut statement = self
+            .connection
+            .prepare_cached("DELETE FROM last_derivation_indices WHERE keychain=:keychain")?;
+        statement.execute(named_params! {
+            ":keychain": &keychain
+        })?;
+
+        Ok(())
+    }
+}
+
+impl BatchOperations for SqliteDatabase {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        self.insert_script_pubkey(keychain, child, script.as_bytes())?;
+        Ok(())
+    }
+
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+        self.insert_utxo(
+            utxo.txout.value,
+            serde_json::to_string(&utxo.keychain)?,
+            utxo.outpoint.vout,
+            &utxo.outpoint.txid,
+            utxo.txout.script_pubkey.as_bytes(),
+        )?;
+        Ok(())
+    }
+
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        match self.select_transaction_by_txid(&transaction.txid())? {
+            Some(_) => {
+                self.update_transaction(&transaction.txid(), &serialize(transaction))?;
+            }
+            None => {
+                self.insert_transaction(&transaction.txid(), &serialize(transaction))?;
+            }
+        }
+        Ok(())
+    }
+
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        match self.select_transaction_details_by_txid(&transaction.txid)? {
+            Some(_) => {
+                self.update_transaction_details(transaction)?;
+            }
+            None => {
+                self.insert_transaction_details(transaction)?;
+            }
+        }
+
+        if let Some(tx) = &transaction.transaction {
+            self.set_raw_tx(tx)?;
+        }
+
+        Ok(())
+    }
+
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        self.update_last_derivation_index(serde_json::to_string(&keychain)?, value)?;
+        Ok(())
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        let script = self.select_script_pubkey_by_path(keychain.clone(), child)?;
+        match script {
+            Some(script) => {
+                self.delete_script_pubkey_by_path(keychain, child)?;
+                Ok(Some(script))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        match self.select_script_pubkey_by_script(script.as_bytes())? {
+            Some((keychain, child)) => {
+                self.delete_script_pubkey_by_script(script.as_bytes())?;
+                Ok(Some((keychain, child)))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        match self.select_utxo_by_outpoint(&outpoint.txid, outpoint.vout)? {
+            Some((value, keychain, script_pubkey)) => {
+                self.delete_utxo_by_outpoint(&outpoint.txid, outpoint.vout)?;
+                Ok(Some(LocalUtxo {
+                    outpoint: *outpoint,
+                    txout: TxOut {
+                        value,
+                        script_pubkey,
+                    },
+                    keychain,
+                }))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        match self.select_transaction_by_txid(txid)? {
+            Some(tx) => {
+                self.delete_transaction_by_txid(txid)?;
+                Ok(Some(tx))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        match self.select_transaction_details_by_txid(txid)? {
+            Some(transaction_details) => {
+                self.delete_transaction_details_by_txid(txid)?;
+
+                if include_raw {
+                    self.delete_transaction_by_txid(txid)?;
+                }
+                Ok(Some(transaction_details))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        match self.select_last_derivation_index_by_keychain(keychain.clone())? {
+            Some(value) => {
+                self.delete_last_derivation_index_by_keychain(keychain)?;
+
+                Ok(Some(value))
+            }
+            None => Ok(None),
+        }
+    }
+}
+
+impl Database for SqliteDatabase {
+    fn check_descriptor_checksum<B: AsRef<[u8]>>(
+        &mut self,
+        keychain: KeychainKind,
+        bytes: B,
+    ) -> Result<(), Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+
+        match self.select_checksum_by_keychain(keychain.clone())? {
+            Some(checksum) => {
+                if checksum == bytes.as_ref().to_vec() {
+                    Ok(())
+                } else {
+                    Err(Error::ChecksumMismatch)
+                }
+            }
+            None => {
+                self.insert_checksum(keychain, bytes.as_ref())?;
+                Ok(())
+            }
+        }
+    }
+
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        match keychain {
+            Some(keychain) => {
+                let keychain = serde_json::to_string(&keychain)?;
+                self.select_script_pubkeys_by_keychain(keychain)
+            }
+            None => self.select_script_pubkeys(),
+        }
+    }
+
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        self.select_utxos()
+    }
+
+    fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error> {
+        self.select_transactions()
+    }
+
+    fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error> {
+        match include_raw {
+            true => self.select_transaction_details_with_raw(),
+            false => self.select_transaction_details(),
+        }
+    }
+
+    fn get_script_pubkey_from_path(
+        &self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        match self.select_script_pubkey_by_path(keychain, child)? {
+            Some(script) => Ok(Some(script)),
+            None => Ok(None),
+        }
+    }
+
+    fn get_path_from_script_pubkey(
+        &self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        match self.select_script_pubkey_by_script(script.as_bytes())? {
+            Some((keychain, child)) => Ok(Some((keychain, child))),
+            None => Ok(None),
+        }
+    }
+
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        match self.select_utxo_by_outpoint(&outpoint.txid, outpoint.vout)? {
+            Some((value, keychain, script_pubkey)) => Ok(Some(LocalUtxo {
+                outpoint: *outpoint,
+                txout: TxOut {
+                    value,
+                    script_pubkey,
+                },
+                keychain,
+            })),
+            None => Ok(None),
+        }
+    }
+
+    fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        match self.select_transaction_by_txid(txid)? {
+            Some(tx) => Ok(Some(tx)),
+            None => Ok(None),
+        }
+    }
+
+    fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error> {
+        match self.select_transaction_details_by_txid(txid)? {
+            Some(mut transaction_details) => {
+                if !include_raw {
+                    transaction_details.transaction = None;
+                }
+                Ok(Some(transaction_details))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let keychain = serde_json::to_string(&keychain)?;
+        let value = self.select_last_derivation_index_by_keychain(keychain)?;
+        Ok(value)
+    }
+
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let keychain_string = serde_json::to_string(&keychain)?;
+        match self.get_last_index(keychain)? {
+            Some(value) => {
+                self.update_last_derivation_index(keychain_string, value + 1)?;
+                Ok(value + 1)
+            }
+            None => {
+                self.insert_last_derivation_index(keychain_string, 0)?;
+                Ok(0)
+            }
+        }
+    }
+
+    fn flush(&mut self) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
+impl BatchDatabase for SqliteDatabase {
+    type Batch = SqliteDatabase;
+
+    fn begin_batch(&self) -> Self::Batch {
+        let db = SqliteDatabase::new(self.path.clone());
+        db.connection.execute("BEGIN TRANSACTION", []).unwrap();
+        db
+    }
+
+    fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> {
+        batch.connection.execute("COMMIT TRANSACTION", [])?;
+        Ok(())
+    }
+}
+
+pub fn get_connection(path: &str) -> Result<Connection, Error> {
+    let connection = Connection::open(path)?;
+    migrate(&connection)?;
+    Ok(connection)
+}
+
+pub fn get_schema_version(conn: &Connection) -> rusqlite::Result<i32> {
+    let statement = conn.prepare_cached("SELECT version FROM version");
+    match statement {
+        Err(rusqlite::Error::SqliteFailure(e, Some(msg))) => {
+            if msg == "no such table: version" {
+                Ok(0)
+            } else {
+                Err(rusqlite::Error::SqliteFailure(e, Some(msg)))
+            }
+        }
+        Ok(mut stmt) => {
+            let mut rows = stmt.query([])?;
+            match rows.next()? {
+                Some(row) => {
+                    let version: i32 = row.get(0)?;
+                    Ok(version)
+                }
+                None => Ok(0),
+            }
+        }
+        _ => Ok(0),
+    }
+}
+
+pub fn set_schema_version(conn: &Connection, version: i32) -> rusqlite::Result<usize> {
+    conn.execute(
+        "UPDATE version SET version=:version",
+        named_params! {":version": version},
+    )
+}
+
+pub fn migrate(conn: &Connection) -> rusqlite::Result<()> {
+    let version = get_schema_version(conn)?;
+    let stmts = &MIGRATIONS[(version as usize)..];
+    let mut i: i32 = version;
+
+    if version == MIGRATIONS.len() as i32 {
+        log::info!("db up to date, no migration needed");
+        return Ok(());
+    }
+
+    for stmt in stmts {
+        let res = conn.execute(stmt, []);
+        if res.is_err() {
+            println!("migration failed on:\n{}\n{:?}", stmt, res);
+            break;
+        }
+
+        i += 1;
+    }
+
+    set_schema_version(conn, i)?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+pub mod test {
+    use crate::database::SqliteDatabase;
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn get_database() -> SqliteDatabase {
+        let time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+        let mut dir = std::env::temp_dir();
+        dir.push(format!("bdk_{}", time.as_nanos()));
+        SqliteDatabase::new(String::from(dir.to_str().unwrap()))
+    }
+
+    #[test]
+    fn test_script_pubkey() {
+        crate::database::test::test_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_batch_script_pubkey() {
+        crate::database::test::test_batch_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_iter_script_pubkey() {
+        crate::database::test::test_iter_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_del_script_pubkey() {
+        crate::database::test::test_del_script_pubkey(get_database());
+    }
+
+    #[test]
+    fn test_utxo() {
+        crate::database::test::test_utxo(get_database());
+    }
+
+    #[test]
+    fn test_raw_tx() {
+        crate::database::test::test_raw_tx(get_database());
+    }
+
+    #[test]
+    fn test_tx() {
+        crate::database::test::test_tx(get_database());
+    }
+
+    #[test]
+    fn test_last_index() {
+        crate::database::test::test_last_index(get_database());
+    }
+}

src/descriptor/checksum.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptor checksum
 //!

src/descriptor/derived.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Derived descriptor keys
 

src/descriptor/dsl.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptors DSL
 
@@ -188,7 +175,7 @@ macro_rules! impl_node_opcode_two {
 #[doc(hidden)]
 #[macro_export]
 macro_rules! impl_node_opcode_three {
-    ( $terminal_variant:ident, $( $inner:tt )* ) => {
+    ( $terminal_variant:ident, $( $inner:tt )* ) => ({
         use $crate::descriptor::CheckMiniscript;
 
         let inner = $crate::fragment_internal!( @t $( $inner )* );
@@ -214,7 +201,7 @@ macro_rules! impl_node_opcode_three {
 
                 Ok((minisc, a_keymap, networks))
             })
-    };
+    });
 }
 
 #[doc(hidden)]
@@ -228,10 +215,11 @@ macro_rules! impl_sortedmulti {
         use $crate::keys::IntoDescriptorKey;
         let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
 
-        let mut keys = vec![];
-        $(
-            keys.push($key.into_descriptor_key());
-        )*
+        let keys = vec![
+            $(
+                $key.into_descriptor_key(),
+            )*
+        ];
 
         keys.into_iter().collect::<Result<Vec<_>, _>>()
             .map_err($crate::descriptor::DescriptorError::Key)
@@ -547,9 +535,7 @@ macro_rules! fragment_internal {
     ( @t , $( $tail:tt )* ) => ({
         $crate::fragment_internal!( @t $( $tail )* )
     });
-    ( @t ) => ({
-        ()
-    });
+    ( @t ) => ({});
 
     // Fallback to calling `fragment!()`
     ( $( $tokens:tt )* ) => ({
@@ -585,8 +571,9 @@ macro_rules! fragment {
     ( pk ( $key:expr ) ) => ({
         $crate::fragment!(c:pk_k ( $key ))
     });
-    ( pk_h ( $key_hash:expr ) ) => ({
-        $crate::impl_leaf_opcode_value!(PkH, $key_hash)
+    ( pk_h ( $key:expr ) ) => ({
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+        $crate::keys::make_pkh($key, &secp)
     });
     ( after ( $value:expr ) ) => ({
         $crate::impl_leaf_opcode_value!(After, $value)
@@ -615,6 +602,9 @@ macro_rules! fragment {
     ( and_or ( $( $inner:tt )* ) ) => ({
         $crate::impl_node_opcode_three!(AndOr, $( $inner )*)
     });
+    ( andor ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_three!(AndOr, $( $inner )*)
+    });
     ( or_b ( $( $inner:tt )* ) ) => ({
         $crate::impl_node_opcode_two!(OrB, $( $inner )*)
     });
@@ -656,10 +646,11 @@ macro_rules! fragment {
         use $crate::keys::IntoDescriptorKey;
         let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
 
-        let mut keys = vec![];
-        $(
-            keys.push($key.into_descriptor_key());
-        )*
+        let keys = vec![
+            $(
+                $key.into_descriptor_key(),
+            )*
+        ];
 
         keys.into_iter().collect::<Result<Vec<_>, _>>()
             .map_err($crate::descriptor::DescriptorError::Key)
@@ -803,6 +794,25 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_fixed_threeop_descriptors() {
+        let redeem_key = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        let move_key = bitcoin::PublicKey::from_str(
+            "032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af",
+        )
+        .unwrap();
+
+        check(
+            descriptor!(sh(wsh(and_or(pk(redeem_key), older(1000), pk(move_key))))),
+            true,
+            true,
+            &["2MypGwr5eQWAWWJtiJgUEToVxc4zuokjQRe"],
+        );
+    }
+
     #[test]
     fn test_bip32_legacy_descriptors() {
         let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
@@ -968,7 +978,7 @@ mod test {
     fn test_valid_networks() {
         let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
-        let desc_key = (xprv, path.clone()).into_descriptor_key().unwrap();
+        let desc_key = (xprv, path).into_descriptor_key().unwrap();
 
         let (_desc, _key_map, valid_networks) = descriptor!(pkh(desc_key)).unwrap();
         assert_eq!(
@@ -978,7 +988,7 @@ mod test {
 
         let xprv = bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi").unwrap();
         let path = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
-        let desc_key = (xprv, path.clone()).into_descriptor_key().unwrap();
+        let desc_key = (xprv, path).into_descriptor_key().unwrap();
 
         let (_desc, _key_map, valid_networks) = descriptor!(wpkh(desc_key)).unwrap();
         assert_eq!(valid_networks, [Bitcoin].iter().cloned().collect());
@@ -1005,12 +1015,9 @@ mod test {
             descriptor!(sh(wsh(multi(2, desc_key1, desc_key2, desc_key3)))).unwrap();
         assert_eq!(key_map.len(), 3);
 
-        let desc_key1: DescriptorKey<Segwitv0> =
-            (xprv1, path1.clone()).into_descriptor_key().unwrap();
-        let desc_key2: DescriptorKey<Segwitv0> =
-            (xprv2, path2.clone()).into_descriptor_key().unwrap();
-        let desc_key3: DescriptorKey<Segwitv0> =
-            (xprv3, path3.clone()).into_descriptor_key().unwrap();
+        let desc_key1: DescriptorKey<Segwitv0> = (xprv1, path1).into_descriptor_key().unwrap();
+        let desc_key2: DescriptorKey<Segwitv0> = (xprv2, path2).into_descriptor_key().unwrap();
+        let desc_key3: DescriptorKey<Segwitv0> = (xprv3, path3).into_descriptor_key().unwrap();
 
         let (key1, _key_map, _valid_networks) = desc_key1.extract(&secp).unwrap();
         let (key2, _key_map, _valid_networks) = desc_key2.extract(&secp).unwrap();
@@ -1026,7 +1033,7 @@ mod test {
         // this compiles
         let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
-        let desc_key: DescriptorKey<Legacy> = (xprv, path.clone()).into_descriptor_key().unwrap();
+        let desc_key: DescriptorKey<Legacy> = (xprv, path).into_descriptor_key().unwrap();
 
         let (desc, _key_map, _valid_networks) = descriptor!(pkh(desc_key)).unwrap();
         assert_eq!(desc.to_string(), "pkh(tpubD6NzVbkrYhZ4WR7a4vY1VT3khMJMeAxVsfq9TBJyJWrNk247zCJtV7AWf6UJP7rAVsn8NNKdJi3gFyKPTmWZS9iukb91xbn2HbFSMQm2igY/0/*)#yrnz9pp2");

src/descriptor/error.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptor errors
 
@@ -28,11 +15,13 @@
 #[derive(Debug)]
 pub enum Error {
     /// Invalid HD Key path, such as having a wildcard but a length != 1
-    InvalidHDKeyPath,
+    InvalidHdKeyPath,
     /// The provided descriptor doesn't match its checksum
     InvalidDescriptorChecksum,
     /// The descriptor contains hardened derivation steps on public extended keys
     HardenedDerivationXpub,
+    /// The descriptor contains multiple keys with the same BIP32 fingerprint
+    DuplicatedKeys,
 
     /// Error thrown while working with [`keys`](crate::keys)
     Key(crate::keys::KeyError),
@@ -43,11 +32,11 @@ pub enum Error {
     InvalidDescriptorCharacter(char),
 
     /// BIP32 error
-    BIP32(bitcoin::util::bip32::Error),
+    Bip32(bitcoin::util::bip32::Error),
     /// Error during base58 decoding
     Base58(bitcoin::util::base58::Error),
     /// Key-related error
-    PK(bitcoin::util::key::Error),
+    Pk(bitcoin::util::key::Error),
     /// Miniscript error
     Miniscript(miniscript::Error),
     /// Hex decoding error
@@ -58,7 +47,7 @@ impl From<crate::keys::KeyError> for Error {
     fn from(key_error: crate::keys::KeyError) -> Error {
         match key_error {
             crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
-            crate::keys::KeyError::BIP32(inner) => Error::BIP32(inner),
+            crate::keys::KeyError::Bip32(inner) => Error::Bip32(inner),
             e => Error::Key(e),
         }
     }
@@ -72,9 +61,9 @@ impl std::fmt::Display for Error {
 
 impl std::error::Error for Error {}
 
-impl_error!(bitcoin::util::bip32::Error, BIP32);
+impl_error!(bitcoin::util::bip32::Error, Bip32);
 impl_error!(bitcoin::util::base58::Error, Base58);
-impl_error!(bitcoin::util::key::Error, PK);
+impl_error!(bitcoin::util::key::Error, Pk);
 impl_error!(miniscript::Error, Miniscript);
 impl_error!(bitcoin::hashes::hex::Error, Hex);
 impl_error!(crate::descriptor::policy::PolicyError, Policy);

src/descriptor/mod.rs

@@ -1,33 +1,20 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptors
 //!
 //! This module contains generic utilities to work with descriptors, plus some re-exported types
 //! from [`miniscript`].
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::ops::Deref;
 
 use bitcoin::util::bip32::{
@@ -40,6 +27,8 @@ use miniscript::descriptor::{DescriptorPublicKey, DescriptorType, DescriptorXKey
 pub use miniscript::{descriptor::KeyMap, Descriptor, Legacy, Miniscript, ScriptContext, Segwitv0};
 use miniscript::{DescriptorTrait, ForEachKey, TranslatePk};
 
+use crate::descriptor::policy::BuildSatisfaction;
+
 pub mod checksum;
 pub(crate) mod derived;
 #[doc(hidden)]
@@ -69,7 +58,7 @@ pub type DerivedDescriptor<'s> = Descriptor<DerivedDescriptorKey<'s>>;
 ///
 /// [`psbt::Input`]: bitcoin::util::psbt::Input
 /// [`psbt::Output`]: bitcoin::util::psbt::Output
-pub type HDKeyPaths = BTreeMap<PublicKey, KeySource>;
+pub type HdKeyPaths = BTreeMap<PublicKey, KeySource>;
 
 /// Trait for types which can be converted into an [`ExtendedDescriptor`] and a [`KeyMap`] usable by a wallet in a specific [`Network`]
 pub trait IntoWalletDescriptor {
@@ -139,11 +128,11 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
             let (pk, _, networks) = if self.0.is_witness() {
                 let desciptor_key: DescriptorKey<miniscript::Segwitv0> =
                     pk.clone().into_descriptor_key()?;
-                desciptor_key.extract(&secp)?
+                desciptor_key.extract(secp)?
             } else {
                 let desciptor_key: DescriptorKey<miniscript::Legacy> =
                     pk.clone().into_descriptor_key()?;
-                desciptor_key.extract(&secp)?
+                desciptor_key.extract(secp)?
             };
 
             if networks.contains(&network) {
@@ -225,6 +214,24 @@ pub(crate) fn into_wallet_descriptor_checked<T: IntoWalletDescriptor>(
         return Err(DescriptorError::HardenedDerivationXpub);
     }
 
+    // Ensure that there are no duplicated keys
+    let mut found_keys = HashSet::new();
+    let descriptor_contains_duplicated_keys = descriptor.for_any_key(|k| {
+        if let DescriptorPublicKey::XPub(xkey) = k.as_key() {
+            let fingerprint = xkey.root_fingerprint(secp);
+            if found_keys.contains(&fingerprint) {
+                return true;
+            }
+
+            found_keys.insert(fingerprint);
+        }
+
+        false
+    });
+    if descriptor_contains_duplicated_keys {
+        return Err(DescriptorError::DuplicatedKeys);
+    }
+
     Ok((descriptor, keymap))
 }
 
@@ -250,6 +257,7 @@ pub trait ExtractPolicy {
     fn extract_policy(
         &self,
         signers: &SignersContainer,
+        psbt: BuildSatisfaction,
         secp: &SecpCtx,
     ) -> Result<Option<Policy>, DescriptorError>;
 }
@@ -324,7 +332,7 @@ impl XKeyUtils for DescriptorXKey<ExtendedPrivKey> {
 }
 
 pub(crate) trait DerivedDescriptorMeta {
-    fn get_hd_keypaths(&self, secp: &SecpCtx) -> Result<HDKeyPaths, DescriptorError>;
+    fn get_hd_keypaths(&self, secp: &SecpCtx) -> Result<HdKeyPaths, DescriptorError>;
 }
 
 pub(crate) trait DescriptorMeta {
@@ -332,7 +340,7 @@ pub(crate) trait DescriptorMeta {
     fn get_extended_keys(&self) -> Result<Vec<DescriptorXKey<ExtendedPubKey>>, DescriptorError>;
     fn derive_from_hd_keypaths<'s>(
         &self,
-        hd_keypaths: &HDKeyPaths,
+        hd_keypaths: &HdKeyPaths,
         secp: &'s SecpCtx,
     ) -> Option<DerivedDescriptor<'s>>;
     fn derive_from_psbt_input<'s>(
@@ -401,7 +409,7 @@ impl DescriptorMeta for ExtendedDescriptor {
 
     fn derive_from_hd_keypaths<'s>(
         &self,
-        hd_keypaths: &HDKeyPaths,
+        hd_keypaths: &HdKeyPaths,
         secp: &'s SecpCtx,
     ) -> Option<DerivedDescriptor<'s>> {
         let index: HashMap<_, _> = hd_keypaths.values().map(|(a, b)| (a, b)).collect();
@@ -500,7 +508,7 @@ impl DescriptorMeta for ExtendedDescriptor {
 }
 
 impl<'s> DerivedDescriptorMeta for DerivedDescriptor<'s> {
-    fn get_hd_keypaths(&self, secp: &SecpCtx) -> Result<HDKeyPaths, DescriptorError> {
+    fn get_hd_keypaths(&self, secp: &SecpCtx) -> Result<HdKeyPaths, DescriptorError> {
         let mut answer = BTreeMap::new();
         self.for_each_key(|key| {
             if let DescriptorPublicKey::XPub(xpub) = key.as_key().deref() {
@@ -532,7 +540,7 @@ mod test {
     use bitcoin::util::{bip32, psbt};
 
     use super::*;
-    use crate::psbt::PSBTUtils;
+    use crate::psbt::PsbtUtils;
 
     #[test]
     fn test_derive_from_psbt_input_wpkh_wif() {
@@ -783,5 +791,14 @@ mod test {
             result.unwrap_err(),
             DescriptorError::HardenedDerivationXpub
         ));
+
+        let descriptor = "wsh(multi(2,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/*))";
+        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+
+        assert!(result.is_err());
+        assert!(matches!(
+            result.unwrap_err(),
+            DescriptorError::DuplicatedKeys
+        ));
     }
 }

src/descriptor/template.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptor templates
 //!
@@ -50,7 +37,7 @@ pub type DescriptorTemplateOut = (ExtendedDescriptor, KeyMap, ValidNetworks);
 ///
 /// ```
 /// use bdk::descriptor::error::Error as DescriptorError;
-/// use bdk::keys::{KeyError, IntoDescriptorKey};
+/// use bdk::keys::{IntoDescriptorKey, KeyError};
 /// use bdk::miniscript::Legacy;
 /// use bdk::template::{DescriptorTemplate, DescriptorTemplateOut};
 ///
@@ -75,7 +62,7 @@ impl<T: DescriptorTemplate> IntoWalletDescriptor for T {
         secp: &SecpCtx,
         network: Network,
     ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
-        Ok(self.build()?.into_wallet_descriptor(secp, network)?)
+        self.build()?.into_wallet_descriptor(secp, network)
     }
 }
 
@@ -87,28 +74,29 @@ impl<T: DescriptorTemplate> IntoWalletDescriptor for T {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::P2PKH;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::P2Pkh;
 ///
 /// let key =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
 /// let wallet = Wallet::new_offline(
-///     P2PKH(key),
+///     P2Pkh(key),
 ///     None,
 ///     Network::Testnet,
 ///     MemoryDatabase::default(),
 /// )?;
 ///
 /// assert_eq!(
-///     wallet.get_new_address()?.to_string(),
+///     wallet.get_address(New)?.to_string(),
 ///     "mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct P2PKH<K: IntoDescriptorKey<Legacy>>(pub K);
+pub struct P2Pkh<K: IntoDescriptorKey<Legacy>>(pub K);
 
-impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for P2PKH<K> {
+impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for P2Pkh<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(descriptor!(pkh(self.0))?)
+        descriptor!(pkh(self.0))
     }
 }
 
@@ -120,29 +108,30 @@ impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for P2PKH<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::P2WPKH_P2SH;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::P2Wpkh_P2Sh;
 ///
 /// let key =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
 /// let wallet = Wallet::new_offline(
-///     P2WPKH_P2SH(key),
+///     P2Wpkh_P2Sh(key),
 ///     None,
 ///     Network::Testnet,
 ///     MemoryDatabase::default(),
 /// )?;
 ///
 /// assert_eq!(
-///     wallet.get_new_address()?.to_string(),
+///     wallet.get_address(New)?.to_string(),
 ///     "2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 #[allow(non_camel_case_types)]
-pub struct P2WPKH_P2SH<K: IntoDescriptorKey<Segwitv0>>(pub K);
+pub struct P2Wpkh_P2Sh<K: IntoDescriptorKey<Segwitv0>>(pub K);
 
-impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH_P2SH<K> {
+impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh_P2Sh<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(descriptor!(sh(wpkh(self.0)))?)
+        descriptor!(sh(wpkh(self.0)))
     }
 }
 
@@ -154,28 +143,29 @@ impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH_P2SH<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::P2WPKH;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::P2Wpkh;
 ///
 /// let key =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
 /// let wallet = Wallet::new_offline(
-///     P2WPKH(key),
+///     P2Wpkh(key),
 ///     None,
 ///     Network::Testnet,
 ///     MemoryDatabase::default(),
 /// )?;
 ///
 /// assert_eq!(
-///     wallet.get_new_address()?.to_string(),
+///     wallet.get_address(New)?.to_string(),
 ///     "tb1q4525hmgw265tl3drrl8jjta7ayffu6jf68ltjd"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct P2WPKH<K: IntoDescriptorKey<Segwitv0>>(pub K);
+pub struct P2Wpkh<K: IntoDescriptorKey<Segwitv0>>(pub K);
 
-impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH<K> {
+impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(descriptor!(wpkh(self.0))?)
+        descriptor!(wpkh(self.0))
     }
 }
 
@@ -183,7 +173,7 @@ impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH<K> {
 ///
 /// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
 ///
-/// See [`BIP44Public`] for a template that can work with a `xpub`/`tpub`.
+/// See [`Bip44Public`] for a template that can work with a `xpub`/`tpub`.
 ///
 /// ## Example
 ///
@@ -192,25 +182,26 @@ impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet,  KeychainKind};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::BIP44;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip44;
 ///
 /// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
 /// let wallet = Wallet::new_offline(
-///     BIP44(key.clone(), KeychainKind::External),
-///     Some(BIP44(key, KeychainKind::Internal)),
+///     Bip44(key.clone(), KeychainKind::External),
+///     Some(Bip44(key, KeychainKind::Internal)),
 ///     Network::Testnet,
 ///     MemoryDatabase::default()
 /// )?;
 ///
-/// assert_eq!(wallet.get_new_address()?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.get_address(New)?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
 /// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#xgaaevjx");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct BIP44<K: DerivableKey<Legacy>>(pub K, pub KeychainKind);
+pub struct Bip44<K: DerivableKey<Legacy>>(pub K, pub KeychainKind);
 
-impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44<K> {
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(P2PKH(legacy::make_bipxx_private(44, self.0, self.1)?).build()?)
+        P2Pkh(legacy::make_bipxx_private(44, self.0, self.1)?).build()
     }
 }
 
@@ -220,7 +211,7 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44<K> {
 ///
 /// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
 ///
-/// See [`BIP44`] for a template that does the full derivation, but requires private data
+/// See [`Bip44`] for a template that does the full derivation, but requires private data
 /// for the key.
 ///
 /// ## Example
@@ -230,26 +221,27 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet,  KeychainKind};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::BIP44Public;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip44Public;
 ///
 /// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
 /// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
 /// let wallet = Wallet::new_offline(
-///     BIP44Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(BIP44Public(key, fingerprint, KeychainKind::Internal)),
+///     Bip44Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(Bip44Public(key, fingerprint, KeychainKind::Internal)),
 ///     Network::Testnet,
 ///     MemoryDatabase::default()
 /// )?;
 ///
-/// assert_eq!(wallet.get_new_address()?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.get_address(New)?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
 /// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#xgaaevjx");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct BIP44Public<K: DerivableKey<Legacy>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+pub struct Bip44Public<K: DerivableKey<Legacy>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
 
-impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44Public<K> {
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44Public<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(P2PKH(legacy::make_bipxx_public(44, self.0, self.1, self.2)?).build()?)
+        P2Pkh(legacy::make_bipxx_public(44, self.0, self.1, self.2)?).build()
     }
 }
 
@@ -257,7 +249,7 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44Public<K> {
 ///
 /// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
 ///
-/// See [`BIP49Public`] for a template that can work with a `xpub`/`tpub`.
+/// See [`Bip49Public`] for a template that can work with a `xpub`/`tpub`.
 ///
 /// ## Example
 ///
@@ -266,25 +258,26 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44Public<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet,  KeychainKind};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::BIP49;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip49;
 ///
 /// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
 /// let wallet = Wallet::new_offline(
-///     BIP49(key.clone(), KeychainKind::External),
-///     Some(BIP49(key, KeychainKind::Internal)),
+///     Bip49(key.clone(), KeychainKind::External),
+///     Some(Bip49(key, KeychainKind::Internal)),
 ///     Network::Testnet,
 ///     MemoryDatabase::default()
 /// )?;
 ///
-/// assert_eq!(wallet.get_new_address()?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.get_address(New)?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
 /// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#gsmdv4xr");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct BIP49<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+pub struct Bip49<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
 
-impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49<K> {
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(P2WPKH_P2SH(segwit_v0::make_bipxx_private(49, self.0, self.1)?).build()?)
+        P2Wpkh_P2Sh(segwit_v0::make_bipxx_private(49, self.0, self.1)?).build()
     }
 }
 
@@ -294,7 +287,7 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49<K> {
 ///
 /// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
 ///
-/// See [`BIP49`] for a template that does the full derivation, but requires private data
+/// See [`Bip49`] for a template that does the full derivation, but requires private data
 /// for the key.
 ///
 /// ## Example
@@ -304,26 +297,27 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet,  KeychainKind};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::BIP49Public;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip49Public;
 ///
 /// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
 /// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
 /// let wallet = Wallet::new_offline(
-///     BIP49Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(BIP49Public(key, fingerprint, KeychainKind::Internal)),
+///     Bip49Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(Bip49Public(key, fingerprint, KeychainKind::Internal)),
 ///     Network::Testnet,
 ///     MemoryDatabase::default()
 /// )?;
 ///
-/// assert_eq!(wallet.get_new_address()?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.get_address(New)?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
 /// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#gsmdv4xr");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct BIP49Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+pub struct Bip49Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
 
-impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49Public<K> {
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49Public<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(P2WPKH_P2SH(segwit_v0::make_bipxx_public(49, self.0, self.1, self.2)?).build()?)
+        P2Wpkh_P2Sh(segwit_v0::make_bipxx_public(49, self.0, self.1, self.2)?).build()
     }
 }
 
@@ -331,7 +325,7 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49Public<K> {
 ///
 /// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
 ///
-/// See [`BIP84Public`] for a template that can work with a `xpub`/`tpub`.
+/// See [`Bip84Public`] for a template that can work with a `xpub`/`tpub`.
 ///
 /// ## Example
 ///
@@ -340,25 +334,26 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49Public<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet,  KeychainKind};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::BIP84;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip84;
 ///
 /// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
 /// let wallet = Wallet::new_offline(
-///     BIP84(key.clone(), KeychainKind::External),
-///     Some(BIP84(key, KeychainKind::Internal)),
+///     Bip84(key.clone(), KeychainKind::External),
+///     Some(Bip84(key, KeychainKind::Internal)),
 ///     Network::Testnet,
 ///     MemoryDatabase::default()
 /// )?;
 ///
-/// assert_eq!(wallet.get_new_address()?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.get_address(New)?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
 /// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#nkk5dtkg");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct BIP84<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+pub struct Bip84<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
 
-impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84<K> {
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(P2WPKH(segwit_v0::make_bipxx_private(84, self.0, self.1)?).build()?)
+        P2Wpkh(segwit_v0::make_bipxx_private(84, self.0, self.1)?).build()
     }
 }
 
@@ -368,7 +363,7 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84<K> {
 ///
 /// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
 ///
-/// See [`BIP84`] for a template that does the full derivation, but requires private data
+/// See [`Bip84`] for a template that does the full derivation, but requires private data
 /// for the key.
 ///
 /// ## Example
@@ -378,26 +373,27 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84<K> {
 /// # use bdk::bitcoin::{PrivateKey, Network};
 /// # use bdk::{Wallet,  KeychainKind};
 /// # use bdk::database::MemoryDatabase;
-/// use bdk::template::BIP84Public;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip84Public;
 ///
 /// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
 /// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
 /// let wallet = Wallet::new_offline(
-///     BIP84Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(BIP84Public(key, fingerprint, KeychainKind::Internal)),
+///     Bip84Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(Bip84Public(key, fingerprint, KeychainKind::Internal)),
 ///     Network::Testnet,
 ///     MemoryDatabase::default()
 /// )?;
 ///
-/// assert_eq!(wallet.get_new_address()?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.get_address(New)?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
 /// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#nkk5dtkg");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
-pub struct BIP84Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+pub struct Bip84Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
 
-impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84Public<K> {
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84Public<K> {
     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
-        Ok(P2WPKH(segwit_v0::make_bipxx_public(84, self.0, self.1, self.2)?).build()?)
+        P2Wpkh(segwit_v0::make_bipxx_public(84, self.0, self.1, self.2)?).build()
     }
 }
 
@@ -440,11 +436,11 @@ macro_rules! expand_make_bipxx {
                     KeychainKind::Internal => vec![bip32::ChildNumber::from_normal_idx(1)?].into(),
                 };
 
-                let mut source_path = Vec::with_capacity(3);
-                source_path.push(bip32::ChildNumber::from_hardened_idx(bip)?);
-                source_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
-                source_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
-                let source_path: bip32::DerivationPath = source_path.into();
+                let source_path = bip32::DerivationPath::from(vec![
+                    bip32::ChildNumber::from_hardened_idx(bip)?,
+                    bip32::ChildNumber::from_hardened_idx(0)?,
+                    bip32::ChildNumber::from_hardened_idx(0)?,
+                ]);
 
                 Ok((key, (parent_fingerprint, source_path), derivation_path))
             }
@@ -459,11 +455,12 @@ expand_make_bipxx!(segwit_v0, Segwitv0);
 mod test {
     // test existing descriptor templates, make sure they are expanded to the right descriptors
 
+    use std::str::FromStr;
+
     use super::*;
     use crate::descriptor::derived::AsDerived;
     use crate::descriptor::{DescriptorError, DescriptorMeta};
     use crate::keys::ValidNetworks;
-    use bitcoin::hashes::core::str::FromStr;
     use bitcoin::network::constants::Network::Regtest;
     use bitcoin::secp256k1::Secp256k1;
     use miniscript::descriptor::{DescriptorPublicKey, DescriptorTrait, KeyMap};
@@ -500,7 +497,7 @@ mod test {
             bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
                 .unwrap();
         check(
-            P2PKH(prvkey).build(),
+            P2Pkh(prvkey).build(),
             false,
             true,
             &["mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"],
@@ -511,7 +508,7 @@ mod test {
         )
         .unwrap();
         check(
-            P2PKH(pubkey).build(),
+            P2Pkh(pubkey).build(),
             false,
             true,
             &["muZpTpBYhxmRFuCjLc7C6BBDF32C8XVJUi"],
@@ -525,7 +522,7 @@ mod test {
             bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
                 .unwrap();
         check(
-            P2WPKH_P2SH(prvkey).build(),
+            P2Wpkh_P2Sh(prvkey).build(),
             true,
             true,
             &["2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"],
@@ -536,7 +533,7 @@ mod test {
         )
         .unwrap();
         check(
-            P2WPKH_P2SH(pubkey).build(),
+            P2Wpkh_P2Sh(pubkey).build(),
             true,
             true,
             &["2N5LiC3CqzxDamRTPG1kiNv1FpNJQ7x28sb"],
@@ -550,7 +547,7 @@ mod test {
             bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
                 .unwrap();
         check(
-            P2WPKH(prvkey).build(),
+            P2Wpkh(prvkey).build(),
             true,
             true,
             &["bcrt1q4525hmgw265tl3drrl8jjta7ayffu6jfcwxx9y"],
@@ -561,7 +558,7 @@ mod test {
         )
         .unwrap();
         check(
-            P2WPKH(pubkey).build(),
+            P2Wpkh(pubkey).build(),
             true,
             true,
             &["bcrt1qngw83fg8dz0k749cg7k3emc7v98wy0c7azaa6h"],
@@ -573,7 +570,7 @@ mod test {
     fn test_bip44_template() {
         let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
-            BIP44(prvkey, KeychainKind::External).build(),
+            Bip44(prvkey, KeychainKind::External).build(),
             false,
             false,
             &[
@@ -583,7 +580,7 @@ mod test {
             ],
         );
         check(
-            BIP44(prvkey, KeychainKind::Internal).build(),
+            Bip44(prvkey, KeychainKind::Internal).build(),
             false,
             false,
             &[
@@ -600,7 +597,7 @@ mod test {
         let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
         let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
-            BIP44Public(pubkey, fingerprint, KeychainKind::External).build(),
+            Bip44Public(pubkey, fingerprint, KeychainKind::External).build(),
             false,
             false,
             &[
@@ -610,7 +607,7 @@ mod test {
             ],
         );
         check(
-            BIP44Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            Bip44Public(pubkey, fingerprint, KeychainKind::Internal).build(),
             false,
             false,
             &[
@@ -626,7 +623,7 @@ mod test {
     fn test_bip49_template() {
         let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
-            BIP49(prvkey, KeychainKind::External).build(),
+            Bip49(prvkey, KeychainKind::External).build(),
             true,
             false,
             &[
@@ -636,7 +633,7 @@ mod test {
             ],
         );
         check(
-            BIP49(prvkey, KeychainKind::Internal).build(),
+            Bip49(prvkey, KeychainKind::Internal).build(),
             true,
             false,
             &[
@@ -653,7 +650,7 @@ mod test {
         let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
         let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
-            BIP49Public(pubkey, fingerprint, KeychainKind::External).build(),
+            Bip49Public(pubkey, fingerprint, KeychainKind::External).build(),
             true,
             false,
             &[
@@ -663,7 +660,7 @@ mod test {
             ],
         );
         check(
-            BIP49Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            Bip49Public(pubkey, fingerprint, KeychainKind::Internal).build(),
             true,
             false,
             &[
@@ -679,7 +676,7 @@ mod test {
     fn test_bip84_template() {
         let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
-            BIP84(prvkey, KeychainKind::External).build(),
+            Bip84(prvkey, KeychainKind::External).build(),
             true,
             false,
             &[
@@ -689,7 +686,7 @@ mod test {
             ],
         );
         check(
-            BIP84(prvkey, KeychainKind::Internal).build(),
+            Bip84(prvkey, KeychainKind::Internal).build(),
             true,
             false,
             &[
@@ -706,7 +703,7 @@ mod test {
         let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
         let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
-            BIP84Public(pubkey, fingerprint, KeychainKind::External).build(),
+            Bip84Public(pubkey, fingerprint, KeychainKind::External).build(),
             true,
             false,
             &[
@@ -716,7 +713,7 @@ mod test {
             ],
         );
         check(
-            BIP84Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            Bip84Public(pubkey, fingerprint, KeychainKind::Internal).build(),
             true,
             false,
             &[

src/doctest.rs

@@ -1,3 +1,14 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
 #[doc(include = "../README.md")]
 #[cfg(doctest)]
 pub struct ReadmeDoctests;

src/error.rs

@@ -1,29 +1,17 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::fmt;
 
+use crate::bitcoin::Network;
 use crate::{descriptor, wallet, wallet::address_validator};
 use bitcoin::OutPoint;
 
@@ -36,10 +24,6 @@ pub enum Error {
     Generic(String),
     /// This error is thrown when trying to convert Bare and Public key script to address
     ScriptDoesntHaveAddressForm,
-    /// Found multiple outputs when `single_recipient` option has been specified
-    SingleRecipientMultipleOutputs,
-    /// `single_recipient` option is selected but neither `drain_wallet` nor `manually_selected_only` are
-    SingleRecipientNoInputs,
     /// Cannot build a tx without recipients
     NoRecipients,
     /// `manually_selected_only` option is selected but no utxo has been passed
@@ -60,7 +44,7 @@ pub enum Error {
     /// the desired outputs plus fee, if there is not such combination this error is thrown
     BnBNoExactMatch,
     /// Happens when trying to spend an UTXO that is not in the internal database
-    UnknownUTXO,
+    UnknownUtxo,
     /// Thrown when a tx is not found in the internal database
     TransactionNotFound,
     /// Happens when trying to bump a transaction that is already confirmed
@@ -77,6 +61,8 @@ pub enum Error {
         /// Required fee absolute value (satoshi)
         required: u64,
     },
+    /// Node doesn't have data to estimate a fee rate
+    FeeRateUnavailable,
     /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
     /// key in the descriptor must either be a master key itself (having depth = 0) or have an
     /// explicit origin provided
@@ -93,6 +79,16 @@ pub enum Error {
     InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
     /// Signing error
     Signer(crate::wallet::signer::SignerError),
+    /// Invalid network
+    InvalidNetwork {
+        /// requested network, for example what is given as bdk-cli option
+        requested: Network,
+        /// found network, for example the network of the bitcoin node
+        found: Network,
+    },
+    #[cfg(feature = "verify")]
+    /// Transaction verification error
+    Verification(crate::wallet::verify::VerifyError),
 
     /// Progress value must be between `0.0` (included) and `100.0` (included)
     InvalidProgressValue(f32),
@@ -110,15 +106,17 @@ pub enum Error {
     /// Miniscript error
     Miniscript(miniscript::Error),
     /// BIP32 error
-    BIP32(bitcoin::util::bip32::Error),
+    Bip32(bitcoin::util::bip32::Error),
     /// An ECDSA error
     Secp256k1(bitcoin::secp256k1::Error),
     /// Error serializing or deserializing JSON data
-    JSON(serde_json::Error),
+    Json(serde_json::Error),
     /// Hex decoding error
     Hex(bitcoin::hashes::hex::Error),
     /// Partially signed bitcoin transaction error
-    PSBT(bitcoin::util::psbt::Error),
+    Psbt(bitcoin::util::psbt::Error),
+    /// Partially signed bitcoin transaction parseerror
+    PsbtParse(bitcoin::util::psbt::PsbtParseError),
 
     //KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
     //MissingInputUTXO(usize),
@@ -132,13 +130,19 @@ pub enum Error {
     Electrum(electrum_client::Error),
     #[cfg(feature = "esplora")]
     /// Esplora client error
-    Esplora(crate::blockchain::esplora::EsploraError),
+    Esplora(Box<crate::blockchain::esplora::EsploraError>),
     #[cfg(feature = "compact_filters")]
     /// Compact filters client error)
     CompactFilters(crate::blockchain::compact_filters::CompactFiltersError),
     #[cfg(feature = "key-value-db")]
     /// Sled database error
     Sled(sled::Error),
+    #[cfg(feature = "rpc")]
+    /// Rpc client error
+    Rpc(bitcoincore_rpc::Error),
+    #[cfg(feature = "sqlite")]
+    /// Rusqlite client error
+    Rusqlite(rusqlite::Error),
 }
 
 impl fmt::Display for Error {
@@ -171,7 +175,7 @@ impl From<crate::keys::KeyError> for Error {
     fn from(key_error: crate::keys::KeyError) -> Error {
         match key_error {
             crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
-            crate::keys::KeyError::BIP32(inner) => Error::BIP32(inner),
+            crate::keys::KeyError::Bip32(inner) => Error::Bip32(inner),
             crate::keys::KeyError::InvalidChecksum => Error::ChecksumMismatch,
             e => Error::Key(e),
         }
@@ -180,18 +184,21 @@ impl From<crate::keys::KeyError> for Error {
 
 impl_error!(bitcoin::consensus::encode::Error, Encode);
 impl_error!(miniscript::Error, Miniscript);
-impl_error!(bitcoin::util::bip32::Error, BIP32);
+impl_error!(bitcoin::util::bip32::Error, Bip32);
 impl_error!(bitcoin::secp256k1::Error, Secp256k1);
-impl_error!(serde_json::Error, JSON);
+impl_error!(serde_json::Error, Json);
 impl_error!(bitcoin::hashes::hex::Error, Hex);
-impl_error!(bitcoin::util::psbt::Error, PSBT);
+impl_error!(bitcoin::util::psbt::Error, Psbt);
+impl_error!(bitcoin::util::psbt::PsbtParseError, PsbtParse);
 
 #[cfg(feature = "electrum")]
 impl_error!(electrum_client::Error, Electrum);
-#[cfg(feature = "esplora")]
-impl_error!(crate::blockchain::esplora::EsploraError, Esplora);
 #[cfg(feature = "key-value-db")]
 impl_error!(sled::Error, Sled);
+#[cfg(feature = "rpc")]
+impl_error!(bitcoincore_rpc::Error, Rpc);
+#[cfg(feature = "sqlite")]
+impl_error!(rusqlite::Error, Rusqlite);
 
 #[cfg(feature = "compact_filters")]
 impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
@@ -202,3 +209,20 @@ impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
         }
     }
 }
+
+#[cfg(feature = "verify")]
+impl From<crate::wallet::verify::VerifyError> for Error {
+    fn from(other: crate::wallet::verify::VerifyError) -> Self {
+        match other {
+            crate::wallet::verify::VerifyError::Global(inner) => *inner,
+            err => Error::Verification(err),
+        }
+    }
+}
+
+#[cfg(feature = "esplora")]
+impl From<crate::blockchain::esplora::EsploraError> for Error {
+    fn from(other: crate::blockchain::esplora::EsploraError) -> Self {
+        Error::Esplora(Box::new(other))
+    }
+}

src/keys/bip39.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! BIP-0039
 
@@ -53,7 +40,7 @@ pub type MnemonicWithPassphrase = (Mnemonic, Option<String>);
 #[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
 impl<Ctx: ScriptContext> DerivableKey<Ctx> for Seed {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-        Ok(bip32::ExtendedPrivKey::new_master(Network::Bitcoin, &self.as_bytes())?.into())
+        Ok(bip32::ExtendedPrivKey::new_master(Network::Bitcoin, self.as_bytes())?.into())
     }
 
     fn into_descriptor_key(

src/keys/mod.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Key formats
 
@@ -205,7 +192,7 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// ```
 /// use bdk::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{DescriptorKey, KeyError, ScriptContext, IntoDescriptorKey};
+/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError, ScriptContext};
 ///
 /// pub struct MyKeyType {
 ///     pubkey: PublicKey,
@@ -224,8 +211,8 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// use bdk::bitcoin::PublicKey;
 ///
 /// use bdk::keys::{
-///     mainnet_network, DescriptorKey, DescriptorPublicKey, DescriptorSinglePub, KeyError,
-///     ScriptContext, IntoDescriptorKey,
+///     mainnet_network, DescriptorKey, DescriptorPublicKey, DescriptorSinglePub,
+///     IntoDescriptorKey, KeyError, ScriptContext,
 /// };
 ///
 /// pub struct MyKeyType {
@@ -250,7 +237,7 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// ```
 /// use bdk::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{DescriptorKey, ExtScriptContext, KeyError, ScriptContext, IntoDescriptorKey};
+/// use bdk::keys::{DescriptorKey, ExtScriptContext, IntoDescriptorKey, KeyError, ScriptContext};
 ///
 /// pub struct MyKeyType {
 ///     is_legacy: bool,
@@ -279,7 +266,7 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// use bdk::bitcoin::PublicKey;
 /// use std::str::FromStr;
 ///
-/// use bdk::keys::{DescriptorKey, KeyError, IntoDescriptorKey};
+/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError};
 ///
 /// pub struct MySegwitOnlyKeyType {
 ///     pubkey: PublicKey,
@@ -733,7 +720,7 @@ fn expand_multi_keys<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
 ) -> Result<(Vec<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError> {
     let (pks, key_maps_networks): (Vec<_>, Vec<_>) = pks
         .into_iter()
-        .map(|key| Ok::<_, KeyError>(key.into_descriptor_key()?.extract(secp)?))
+        .map(|key| key.into_descriptor_key()?.extract(secp))
         .collect::<Result<Vec<_>, _>>()?
         .into_iter()
         .map(|(a, b, c)| (a, (b, c)))
@@ -766,6 +753,20 @@ pub fn make_pk<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((minisc, key_map, valid_networks))
 }
 
+// Used internally by `bdk::fragment!` to build `pk_h()` fragments
+#[doc(hidden)]
+pub fn make_pkh<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    descriptor_key: Pk,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), DescriptorError> {
+    let (key, key_map, valid_networks) = descriptor_key.into_descriptor_key()?.extract(secp)?;
+    let minisc = Miniscript::from_ast(Terminal::PkH(key))?;
+
+    minisc.check_minsicript()?;
+
+    Ok((minisc, key_map, valid_networks))
+}
+
 // Used internally by `bdk::fragment!` to build `multi()` fragments
 #[doc(hidden)]
 pub fn make_multi<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
@@ -886,13 +887,13 @@ pub enum KeyError {
     Message(String),
 
     /// BIP32 error
-    BIP32(bitcoin::util::bip32::Error),
+    Bip32(bitcoin::util::bip32::Error),
     /// Miniscript error
     Miniscript(miniscript::Error),
 }
 
 impl_error!(miniscript::Error, Miniscript, KeyError);
-impl_error!(bitcoin::util::bip32::Error, BIP32, KeyError);
+impl_error!(bitcoin::util::bip32::Error, Bip32, KeyError);
 
 impl std::fmt::Display for KeyError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

src/lib.rs

@@ -1,35 +1,19 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 // rustdoc will warn if there are missing docs
 #![warn(missing_docs)]
 // only enables the `doc_cfg` feature when
 // the `docsrs` configuration attribute is defined
 #![cfg_attr(docsrs, feature(doc_cfg))]
-// only enables the nightly `external_doc` feature when
-// `test-md-docs` is enabled
-#![cfg_attr(feature = "test-md-docs", feature(external_doc))]
 
 //! A modern, lightweight, descriptor-based wallet library written in Rust.
 //!
@@ -56,36 +40,39 @@
 //! interact with the bitcoin P2P network.
 //!
 //! ```toml
-//! bdk = "0.4.0"
-//! ```
-//!
-//! ## Sync the balance of a descriptor
-//!
-//! ### Example
-//! ```ignore
-//! use bdk::Wallet;
-//! use bdk::database::MemoryDatabase;
-//! use bdk::blockchain::{noop_progress, ElectrumBlockchain};
-//!
-//! use bdk::electrum_client::Client;
-//!
-//! fn main() -> Result<(), bdk::Error> {
-//!     let client = Client::new("ssl://electrum.blockstream.info:60002")?;
-//!     let wallet = Wallet::new(
-//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
-//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
-//!         bitcoin::Network::Testnet,
-//!         MemoryDatabase::default(),
-//!         ElectrumBlockchain::from(client)
-//!     )?;
-//!
-//!     wallet.sync(noop_progress(), None)?;
-//!
-//!     println!("Descriptor balance: {} SAT", wallet.get_balance()?);
-//!
-//!     Ok(())
-//! }
+//! bdk = "0.13.0"
 //! ```
+#![cfg_attr(
+    feature = "electrum",
+    doc = r##"
+## Sync the balance of a descriptor
+
+### Example
+```no_run
+use bdk::Wallet;
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+use bdk::electrum_client::Client;
+
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+    let wallet = Wallet::new(
+        "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+        ElectrumBlockchain::from(client)
+    )?;
+
+    wallet.sync(noop_progress(), None)?;
+
+    println!("Descriptor balance: {} SAT", wallet.get_balance()?);
+
+    Ok(())
+}
+```
+"##
+)]
 //!
 //! ## Generate a few addresses
 //!
@@ -93,72 +80,80 @@
 //! ```
 //! use bdk::{Wallet};
 //! use bdk::database::MemoryDatabase;
+//! use bdk::wallet::AddressIndex::New;
 //!
 //! fn main() -> Result<(), bdk::Error> {
-//!     let wallet = Wallet::new_offline(
+//! let wallet = Wallet::new_offline(
 //!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
 //!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
 //!         bitcoin::Network::Testnet,
 //!         MemoryDatabase::default(),
 //!     )?;
 //!
-//!     println!("Address #0: {}", wallet.get_new_address()?);
-//!     println!("Address #1: {}", wallet.get_new_address()?);
-//!     println!("Address #2: {}", wallet.get_new_address()?);
-//!
-//!     Ok(())
-//! }
-//! ```
-//!
-//! ## Create a transaction
-//!
-//! ### Example
-//! ```ignore
-//! use base64::decode;
-//! use bdk::{FeeRate, Wallet};
-//! use bdk::database::MemoryDatabase;
-//! use bdk::blockchain::{noop_progress, ElectrumBlockchain};
-//!
-//! use bdk::electrum_client::Client;
-//!
-//! use bitcoin::consensus::serialize;
-//!
-//! fn main() -> Result<(), bdk::Error> {
-//!     let client = Client::new("ssl://electrum.blockstream.info:60002")?;
-//!     let wallet = Wallet::new(
-//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
-//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
-//!         bitcoin::Network::Testnet,
-//!         MemoryDatabase::default(),
-//!         ElectrumBlockchain::from(client)
-//!     )?;
-//!
-//!     wallet.sync(noop_progress(), None)?;
-//!
-//!     let send_to = wallet.get_new_address()?;
-//!     let (psbt, details) = wallet.build_tx()
-//!         .add_recipient(send_to.script_pubkey(), 50_000)
-//!         .enable_rbf()
-//!         .do_not_spend_change()
-//!         .fee_rate(FeeRate::from_sat_per_vb(5.0))
-//!         .finish()?;
-//!
-//!     println!("Transaction details: {:#?}", details);
-//!     println!("Unsigned PSBT: {}", base64::encode(&serialize(&psbt)));
+//!     println!("Address #0: {}", wallet.get_address(New)?);
+//!     println!("Address #1: {}", wallet.get_address(New)?);
+//!     println!("Address #2: {}", wallet.get_address(New)?);
 //!
 //!     Ok(())
 //! }
 //! ```
+#![cfg_attr(
+    feature = "electrum",
+    doc = r##"
+## Create a transaction
+
+### Example
+```no_run
+use bdk::{FeeRate, Wallet};
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+use bdk::electrum_client::Client;
+
+use bitcoin::consensus::serialize;
+use bdk::wallet::AddressIndex::New;
+
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+    let wallet = Wallet::new(
+        "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+        ElectrumBlockchain::from(client)
+    )?;
+
+    wallet.sync(noop_progress(), None)?;
+
+    let send_to = wallet.get_address(New)?;
+    let (psbt, details) = {
+        let mut builder =  wallet.build_tx();
+        builder
+            .add_recipient(send_to.script_pubkey(), 50_000)
+            .enable_rbf()
+            .do_not_spend_change()
+            .fee_rate(FeeRate::from_sat_per_vb(5.0));
+        builder.finish()?
+    };
+
+    println!("Transaction details: {:#?}", details);
+    println!("Unsigned PSBT: {}", &psbt);
+
+    Ok(())
+}
+```
+"##
+)]
 //!
 //! ## Sign a transaction
 //!
 //! ### Example
-//! ```ignore
-//! use base64::decode;
-//! use bdk::{Wallet};
-//! use bdk::database::MemoryDatabase;
+//! ```no_run
+//! use std::str::FromStr;
 //!
-//! use bitcoin::consensus::deserialize;
+//! use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
+//!
+//! use bdk::{Wallet, SignOptions};
+//! use bdk::database::MemoryDatabase;
 //!
 //! fn main() -> Result<(), bdk::Error> {
 //!     let wallet = Wallet::new_offline(
@@ -169,9 +164,9 @@
 //!     )?;
 //!
 //!     let psbt = "...";
-//!     let psbt = deserialize(&base64::decode(psbt).unwrap())?;
+//!     let mut psbt = Psbt::from_str(psbt)?;
 //!
-//!     let (signed_psbt, finalized) = wallet.sign(psbt, None)?;
+//!     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
 //!
 //!     Ok(())
 //! }
@@ -210,11 +205,24 @@ extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
+#[cfg(all(feature = "reqwest", feature = "ureq"))]
+compile_error!("Features reqwest and ureq are mutually exclusive and cannot be enabled together");
+
 #[cfg(all(feature = "async-interface", feature = "electrum"))]
 compile_error!(
     "Features async-interface and electrum are mutually exclusive and cannot be enabled together"
 );
 
+#[cfg(all(feature = "async-interface", feature = "ureq"))]
+compile_error!(
+    "Features async-interface and ureq are mutually exclusive and cannot be enabled together"
+);
+
+#[cfg(all(feature = "async-interface", feature = "compact_filters"))]
+compile_error!(
+    "Features async-interface and compact_filters are mutually exclusive and cannot be enabled together"
+);
+
 #[cfg(feature = "keys-bip39")]
 extern crate bip39;
 
@@ -225,31 +233,21 @@ extern crate async_trait;
 extern crate bdk_macros;
 
 #[cfg(feature = "compact_filters")]
-#[macro_use]
 extern crate lazy_static;
 
+#[cfg(feature = "rpc")]
+pub extern crate bitcoincore_rpc;
+
 #[cfg(feature = "electrum")]
 pub extern crate electrum_client;
 
-#[cfg(feature = "esplora")]
-pub extern crate reqwest;
-
 #[cfg(feature = "key-value-db")]
 pub extern crate sled;
 
-#[allow(unused_imports)]
-#[cfg(test)]
-#[macro_use]
-extern crate testutils;
-#[allow(unused_imports)]
-#[cfg(test)]
-#[macro_use]
-extern crate testutils_macros;
-#[allow(unused_imports)]
-#[cfg(test)]
-#[macro_use]
-extern crate serial_test;
+#[cfg(feature = "sqlite")]
+pub extern crate rusqlite;
 
+#[allow(unused_imports)]
 #[macro_use]
 pub(crate) mod error;
 pub mod blockchain;
@@ -263,11 +261,12 @@ pub(crate) mod types;
 pub mod wallet;
 
 pub use descriptor::template;
-pub use descriptor::HDKeyPaths;
+pub use descriptor::HdKeyPaths;
 pub use error::Error;
 pub use types::*;
 pub use wallet::address_validator;
 pub use wallet::signer;
+pub use wallet::signer::SignOptions;
 pub use wallet::tx_builder::TxBuilder;
 pub use wallet::Wallet;
 
@@ -275,3 +274,10 @@ pub use wallet::Wallet;
 pub fn version() -> &'static str {
     env!("CARGO_PKG_VERSION", "unknown")
 }
+
+// We should consider putting this under a feature flag but we need the macro in doctets so we need
+// to wait until https://github.com/rust-lang/rust/issues/67295 is fixed.
+//
+// Stuff in here is too rough to document atm
+#[doc(hidden)]
+pub mod testutils;

src/psbt/mod.rs

@@ -1,35 +1,23 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
-use bitcoin::util::psbt::PartiallySignedTransaction as PSBT;
+use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
 use bitcoin::TxOut;
 
-pub trait PSBTUtils {
+pub trait PsbtUtils {
     fn get_utxo_for(&self, input_index: usize) -> Option<TxOut>;
 }
 
-impl PSBTUtils for PSBT {
+impl PsbtUtils for Psbt {
+    #[allow(clippy::all)] // We want to allow `manual_map` but it is too new.
     fn get_utxo_for(&self, input_index: usize) -> Option<TxOut> {
         let tx = &self.global.unsigned_tx;
 
@@ -50,3 +38,85 @@ impl PSBTUtils for PSBT {
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    use crate::bitcoin::TxIn;
+    use crate::psbt::Psbt;
+    use crate::wallet::AddressIndex;
+    use crate::wallet::{get_funded_wallet, test::get_test_wpkh};
+    use crate::SignOptions;
+    use std::str::FromStr;
+
+    // from bip 174
+    const PSBT_STR: &str = "cHNidP8BAKACAAAAAqsJSaCMWvfEm4IS9Bfi8Vqz9cM9zxU4IagTn4d6W3vkAAAAAAD+////qwlJoIxa98SbghL0F+LxWrP1wz3PFTghqBOfh3pbe+QBAAAAAP7///8CYDvqCwAAAAAZdqkUdopAu9dAy+gdmI5x3ipNXHE5ax2IrI4kAAAAAAAAGXapFG9GILVT+glechue4O/p+gOcykWXiKwAAAAAAAEHakcwRAIgR1lmF5fAGwNrJZKJSGhiGDR9iYZLcZ4ff89X0eURZYcCIFMJ6r9Wqk2Ikf/REf3xM286KdqGbX+EhtdVRs7tr5MZASEDXNxh/HupccC1AaZGoqg7ECy0OIEhfKaC3Ibi1z+ogpIAAQEgAOH1BQAAAAAXqRQ1RebjO4MsRwUPJNPuuTycA5SLx4cBBBYAFIXRNTfy4mVAWjTbr6nj3aAfuCMIAAAA";
+
+    #[test]
+    #[should_panic(expected = "InputIndexOutOfRange")]
+    fn test_psbt_malformed_psbt_input_legacy() {
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+        psbt.inputs.push(psbt_bip.inputs[0].clone());
+        let options = SignOptions {
+            trust_witness_utxo: true,
+            ..Default::default()
+        };
+        let _ = wallet.sign(&mut psbt, options).unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "InputIndexOutOfRange")]
+    fn test_psbt_malformed_psbt_input_segwit() {
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+        psbt.inputs.push(psbt_bip.inputs[1].clone());
+        let options = SignOptions {
+            trust_witness_utxo: true,
+            ..Default::default()
+        };
+        let _ = wallet.sign(&mut psbt, options).unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "InputIndexOutOfRange")]
+    fn test_psbt_malformed_tx_input() {
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+        psbt.global.unsigned_tx.input.push(TxIn::default());
+        let options = SignOptions {
+            trust_witness_utxo: true,
+            ..Default::default()
+        };
+        let _ = wallet.sign(&mut psbt, options).unwrap();
+    }
+
+    #[test]
+    fn test_psbt_sign_with_finalized() {
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+
+        // add a finalized input
+        psbt.inputs.push(psbt_bip.inputs[0].clone());
+        psbt.global
+            .unsigned_tx
+            .input
+            .push(psbt_bip.global.unsigned_tx.input[0].clone());
+
+        let _ = wallet.sign(&mut psbt, SignOptions::default()).unwrap();
+    }
+}

src/testutils/blockchain_tests.rs

@@ -0,0 +1,881 @@
+use crate::testutils::TestIncomingTx;
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::sha256d;
+use bitcoin::{Address, Amount, Script, Transaction, Txid};
+pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
+pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
+use core::str::FromStr;
+use electrsd::bitcoind::BitcoinD;
+use electrsd::{bitcoind, ElectrsD};
+pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
+#[allow(unused_imports)]
+use log::{debug, error, info, log_enabled, trace, Level};
+use std::collections::HashMap;
+use std::env;
+use std::ops::Deref;
+use std::time::Duration;
+
+pub struct TestClient {
+    pub bitcoind: BitcoinD,
+    pub electrsd: ElectrsD,
+}
+
+impl TestClient {
+    pub fn new(bitcoind_exe: String, electrs_exe: String) -> Self {
+        debug!("launching {} and {}", &bitcoind_exe, &electrs_exe);
+
+        let mut conf = bitcoind::Conf::default();
+        conf.view_stdout = log_enabled!(Level::Debug);
+        let bitcoind = BitcoinD::with_conf(bitcoind_exe, &conf).unwrap();
+
+        let mut conf = electrsd::Conf::default();
+        conf.view_stderr = log_enabled!(Level::Debug);
+        conf.http_enabled = cfg!(feature = "test-esplora");
+
+        let electrsd = ElectrsD::with_conf(electrs_exe, &bitcoind, &conf).unwrap();
+
+        let node_address = bitcoind.client.get_new_address(None, None).unwrap();
+        bitcoind
+            .client
+            .generate_to_address(101, &node_address)
+            .unwrap();
+
+        let mut test_client = TestClient { bitcoind, electrsd };
+        TestClient::wait_for_block(&mut test_client, 101);
+        test_client
+    }
+
+    fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
+        // wait for electrs to index the tx
+        exponential_backoff_poll(|| {
+            self.electrsd.trigger().unwrap();
+            trace!("wait_for_tx {}", txid);
+
+            self.electrsd
+                .client
+                .script_get_history(monitor_script)
+                .unwrap()
+                .iter()
+                .position(|entry| entry.tx_hash == txid)
+        });
+    }
+
+    fn wait_for_block(&mut self, min_height: usize) {
+        self.electrsd.client.block_headers_subscribe().unwrap();
+
+        loop {
+            let header = exponential_backoff_poll(|| {
+                self.electrsd.trigger().unwrap();
+                self.electrsd.client.ping().unwrap();
+                self.electrsd.client.block_headers_pop().unwrap()
+            });
+            if header.height >= min_height {
+                break;
+            }
+        }
+    }
+
+    pub fn receive(&mut self, meta_tx: TestIncomingTx) -> Txid {
+        assert!(
+            !meta_tx.output.is_empty(),
+            "can't create a transaction with no outputs"
+        );
+
+        let mut map = HashMap::new();
+
+        let mut required_balance = 0;
+        for out in &meta_tx.output {
+            required_balance += out.value;
+            map.insert(out.to_address.clone(), Amount::from_sat(out.value));
+        }
+
+        if self.get_balance(None, None).unwrap() < Amount::from_sat(required_balance) {
+            panic!("Insufficient funds in bitcoind. Please generate a few blocks with: `bitcoin-cli generatetoaddress 10 {}`", self.get_new_address(None, None).unwrap());
+        }
+
+        // FIXME: core can't create a tx with two outputs to the same address
+        let tx = self
+            .create_raw_transaction_hex(&[], &map, meta_tx.locktime, meta_tx.replaceable)
+            .unwrap();
+        let tx = self.fund_raw_transaction(tx, None, None).unwrap();
+        let mut tx: Transaction = deserialize(&tx.hex).unwrap();
+
+        if let Some(true) = meta_tx.replaceable {
+            // for some reason core doesn't set this field right
+            for input in &mut tx.input {
+                input.sequence = 0xFFFFFFFD;
+            }
+        }
+
+        let tx = self
+            .sign_raw_transaction_with_wallet(&serialize(&tx), None, None)
+            .unwrap();
+
+        // broadcast through electrum so that it caches the tx immediately
+
+        let txid = self
+            .electrsd
+            .client
+            .transaction_broadcast(&deserialize(&tx.hex).unwrap())
+            .unwrap();
+        debug!("broadcasted to electrum {}", txid);
+
+        if let Some(num) = meta_tx.min_confirmations {
+            self.generate(num, None);
+        }
+
+        let monitor_script = Address::from_str(&meta_tx.output[0].to_address)
+            .unwrap()
+            .script_pubkey();
+        self.wait_for_tx(txid, &monitor_script);
+
+        debug!("Sent tx: {}", txid);
+
+        txid
+    }
+
+    pub fn bump_fee(&mut self, txid: &Txid) -> Txid {
+        let tx = self.get_raw_transaction_info(txid, None).unwrap();
+        assert!(
+            tx.confirmations.is_none(),
+            "Can't bump tx {} because it's already confirmed",
+            txid
+        );
+
+        let bumped: serde_json::Value = self.call("bumpfee", &[txid.to_string().into()]).unwrap();
+        let new_txid = Txid::from_str(&bumped["txid"].as_str().unwrap().to_string()).unwrap();
+
+        let monitor_script =
+            tx.vout[0].script_pub_key.addresses.as_ref().unwrap()[0].script_pubkey();
+        self.wait_for_tx(new_txid, &monitor_script);
+
+        debug!("Bumped {}, new txid {}", txid, new_txid);
+
+        new_txid
+    }
+
+    pub fn generate_manually(&mut self, txs: Vec<Transaction>) -> String {
+        use bitcoin::blockdata::block::{Block, BlockHeader};
+        use bitcoin::blockdata::script::Builder;
+        use bitcoin::blockdata::transaction::{OutPoint, TxIn, TxOut};
+        use bitcoin::hash_types::{BlockHash, TxMerkleNode};
+
+        let block_template: serde_json::Value = self
+            .call("getblocktemplate", &[json!({"rules": ["segwit"]})])
+            .unwrap();
+        trace!("getblocktemplate: {:#?}", block_template);
+
+        let header = BlockHeader {
+            version: block_template["version"].as_i64().unwrap() as i32,
+            prev_blockhash: BlockHash::from_hex(
+                block_template["previousblockhash"].as_str().unwrap(),
+            )
+            .unwrap(),
+            merkle_root: TxMerkleNode::default(),
+            time: block_template["curtime"].as_u64().unwrap() as u32,
+            bits: u32::from_str_radix(block_template["bits"].as_str().unwrap(), 16).unwrap(),
+            nonce: 0,
+        };
+        debug!("header: {:#?}", header);
+
+        let height = block_template["height"].as_u64().unwrap() as i64;
+        let witness_reserved_value: Vec<u8> = sha256d::Hash::default().as_ref().into();
+        // burn block subsidy and fees, not a big deal
+        let mut coinbase_tx = Transaction {
+            version: 1,
+            lock_time: 0,
+            input: vec![TxIn {
+                previous_output: OutPoint::null(),
+                script_sig: Builder::new().push_int(height).into_script(),
+                sequence: 0xFFFFFFFF,
+                witness: vec![witness_reserved_value],
+            }],
+            output: vec![],
+        };
+
+        let mut txdata = vec![coinbase_tx.clone()];
+        txdata.extend_from_slice(&txs);
+
+        let mut block = Block { header, txdata };
+
+        let witness_root = block.witness_root();
+        let witness_commitment =
+            Block::compute_witness_commitment(&witness_root, &coinbase_tx.input[0].witness[0]);
+
+        // now update and replace the coinbase tx
+        let mut coinbase_witness_commitment_script = vec![0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed];
+        coinbase_witness_commitment_script.extend_from_slice(&witness_commitment);
+
+        coinbase_tx.output.push(TxOut {
+            value: 0,
+            script_pubkey: coinbase_witness_commitment_script.into(),
+        });
+        block.txdata[0] = coinbase_tx;
+
+        // set merkle root
+        let merkle_root = block.merkle_root();
+        block.header.merkle_root = merkle_root;
+
+        assert!(block.check_merkle_root());
+        assert!(block.check_witness_commitment());
+
+        // now do PoW :)
+        let target = block.header.target();
+        while block.header.validate_pow(&target).is_err() {
+            block.header.nonce = block.header.nonce.checked_add(1).unwrap(); // panic if we run out of nonces
+        }
+
+        let block_hex: String = serialize(&block).to_hex();
+        debug!("generated block hex: {}", block_hex);
+
+        self.electrsd.client.block_headers_subscribe().unwrap();
+
+        let submit_result: serde_json::Value =
+            self.call("submitblock", &[block_hex.into()]).unwrap();
+        debug!("submitblock: {:?}", submit_result);
+        assert!(
+            submit_result.is_null(),
+            "submitblock error: {:?}",
+            submit_result.as_str()
+        );
+
+        self.wait_for_block(height as usize);
+
+        block.header.block_hash().to_hex()
+    }
+
+    pub fn generate(&mut self, num_blocks: u64, address: Option<Address>) {
+        let address = address.unwrap_or_else(|| self.get_new_address(None, None).unwrap());
+        let hashes = self.generate_to_address(num_blocks, &address).unwrap();
+        let best_hash = hashes.last().unwrap();
+        let height = self.get_block_info(best_hash).unwrap().height;
+
+        self.wait_for_block(height);
+
+        debug!("Generated blocks to new height {}", height);
+    }
+
+    pub fn invalidate(&mut self, num_blocks: u64) {
+        self.electrsd.client.block_headers_subscribe().unwrap();
+
+        let best_hash = self.get_best_block_hash().unwrap();
+        let initial_height = self.get_block_info(&best_hash).unwrap().height;
+
+        let mut to_invalidate = best_hash;
+        for i in 1..=num_blocks {
+            trace!(
+                "Invalidating block {}/{} ({})",
+                i,
+                num_blocks,
+                to_invalidate
+            );
+
+            self.invalidate_block(&to_invalidate).unwrap();
+            to_invalidate = self.get_best_block_hash().unwrap();
+        }
+
+        self.wait_for_block(initial_height - num_blocks as usize);
+
+        debug!(
+            "Invalidated {} blocks to new height of {}",
+            num_blocks,
+            initial_height - num_blocks as usize
+        );
+    }
+
+    pub fn reorg(&mut self, num_blocks: u64) {
+        self.invalidate(num_blocks);
+        self.generate(num_blocks, None);
+    }
+
+    pub fn get_node_address(&self, address_type: Option<AddressType>) -> Address {
+        Address::from_str(
+            &self
+                .get_new_address(None, address_type)
+                .unwrap()
+                .to_string(),
+        )
+        .unwrap()
+    }
+}
+
+pub fn get_electrum_url() -> String {
+    env::var("BDK_ELECTRUM_URL").unwrap_or_else(|_| "tcp://127.0.0.1:50001".to_string())
+}
+
+impl Deref for TestClient {
+    type Target = RpcClient;
+
+    fn deref(&self) -> &Self::Target {
+        &self.bitcoind.client
+    }
+}
+
+impl Default for TestClient {
+    fn default() -> Self {
+        let bitcoind_exe = env::var("BITCOIND_EXE")
+            .ok()
+            .or(bitcoind::downloaded_exe_path())
+            .expect(
+                "you should provide env var BITCOIND_EXE or specifiy a bitcoind version feature",
+            );
+        let electrs_exe = env::var("ELECTRS_EXE")
+            .ok()
+            .or(electrsd::downloaded_exe_path())
+            .expect(
+                "you should provide env var ELECTRS_EXE or specifiy a electrsd version feature",
+            );
+        Self::new(bitcoind_exe, electrs_exe)
+    }
+}
+
+fn exponential_backoff_poll<T, F>(mut poll: F) -> T
+where
+    F: FnMut() -> Option<T>,
+{
+    let mut delay = Duration::from_millis(64);
+    loop {
+        match poll() {
+            Some(data) => break data,
+            None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0),
+            None => {}
+        }
+
+        std::thread::sleep(delay);
+    }
+}
+
+/// This macro runs blockchain tests against a `Blockchain` implementation. It requires access to a
+/// Bitcoin core wallet via RPC. At the moment you have to dig into the code yourself and look at
+/// the setup required to run the tests yourself.
+#[macro_export]
+macro_rules! bdk_blockchain_tests {
+    (
+     fn $_fn_name:ident ( $( $test_client:ident : &TestClient )? $(,)? ) -> $blockchain:ty $block:block) => {
+        #[cfg(test)]
+        mod bdk_blockchain_tests {
+            use $crate::bitcoin::Network;
+            use $crate::testutils::blockchain_tests::TestClient;
+            use $crate::blockchain::noop_progress;
+            use $crate::database::MemoryDatabase;
+            use $crate::types::KeychainKind;
+            use $crate::{Wallet, FeeRate};
+            use $crate::testutils;
+
+            use super::*;
+
+            #[allow(unused_variables)]
+            fn get_blockchain(test_client: &TestClient) -> $blockchain {
+                $( let $test_client = test_client; )?
+                $block
+            }
+
+            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>), test_client: &TestClient) -> Wallet<$blockchain, MemoryDatabase> {
+                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain(test_client)).unwrap()
+            }
+
+            fn init_single_sig() -> (Wallet<$blockchain, MemoryDatabase>, (String, Option<String>), TestClient) {
+                let _ = env_logger::try_init();
+
+                let descriptors = testutils! {
+                    @descriptors ( "wpkh(Alice)" ) ( "wpkh(Alice)" ) ( @keys ( "Alice" => (@generate_xprv "/44'/0'/0'/0/*", "/44'/0'/0'/1/*") ) )
+                };
+
+                let test_client = TestClient::default();
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
+
+                // rpc need to call import_multi before receiving any tx, otherwise will not see tx in the mempool
+                #[cfg(feature = "test-rpc")]
+                wallet.sync(noop_progress(), None).unwrap();
+
+                (wallet, descriptors, test_client)
+            }
+
+            #[test]
+            fn test_sync_simple() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let tx = testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                };
+                println!("{:?}", tx);
+                let txid = test_client.receive(tx);
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External, "incorrect keychain kind");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time");
+            }
+
+            #[test]
+            fn test_sync_stop_gap_20() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 5) => 50_000 )
+                });
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 25) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 100_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+            }
+
+            #[test]
+            fn test_sync_before_and_after_receive() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 0);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+            }
+
+            #[test]
+            fn test_sync_multiple_outputs_same_tx() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000, (@external descriptors, 5) => 30_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 105_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 3, "incorrect number of unspents");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 105_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
+            }
+
+            #[test]
+            fn test_sync_receive_multi() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 5) => 25_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 2, "incorrect number of unspent");
+            }
+
+            #[test]
+            fn test_sync_address_reuse() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 25_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+            }
+
+            #[test]
+            fn test_sync_receive_rbf_replaced() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) ( @replaceable true )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
+
+                let new_txid = test_client.bump_fee(&txid);
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after bump");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs after bump");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent after bump");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, new_txid, "incorrect txid after bump");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received after bump");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent after bump");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect height after bump");
+            }
+
+            // FIXME: I would like this to be cfg_attr(not(feature = "test-esplora"), ignore) but it
+            // doesn't work for some reason.
+            #[cfg(not(feature = "esplora"))]
+            #[test]
+            fn test_sync_reorg_block() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) ( @confirmations 1 ) ( @replaceable true )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert!(list_tx_item.confirmation_time.is_some(), "incorrect confirmation_time");
+
+                // Invalidate 1 block
+                test_client.invalidate(1);
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after invalidate");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid after invalidate");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time after invalidate");
+            }
+
+            #[test]
+            fn test_sync_after_send() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                println!("{}", descriptors.0);
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey(), 25_000);
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                let tx = psbt.extract_tx();
+                println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
+                wallet.broadcast(tx).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after send");
+
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
+            }
+
+            #[test]
+            fn test_update_confirmation_time_after_generate() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                println!("{}", descriptors.0);
+                let node_addr = test_client.get_node_address(None);
+
+                let received_txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_none());
+
+                test_client.generate(1, Some(node_addr));
+                wallet.sync(noop_progress(), None).unwrap();
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_some());
+
+            }
+
+            #[test]
+            fn test_sync_outgoing_from_scratch() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+                let received_txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey(), 25_000);
+                let (mut psbt, details) = builder.finish().unwrap();
+
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after receive");
+
+                // empty wallet
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
+
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
+                wallet.sync(noop_progress(), None).unwrap();
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+
+                let received = tx_map.get(&received_txid).unwrap();
+                assert_eq!(received.received, 50_000, "incorrect received from receiver");
+                assert_eq!(received.sent, 0, "incorrect sent from receiver");
+
+                let sent = tx_map.get(&sent_txid).unwrap();
+                assert_eq!(sent.received, details.received, "incorrect received from sender");
+                assert_eq!(sent.sent, details.sent, "incorrect sent from sender");
+                assert_eq!(sent.fee.unwrap_or(0), details.fee.unwrap_or(0), "incorrect fees from sender");
+            }
+
+            #[test]
+            fn test_sync_long_change_chain() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut total_sent = 0;
+                for _ in 0..5 {
+                    let mut builder = wallet.build_tx();
+                    builder.add_recipient(node_addr.script_pubkey(), 5_000);
+                    let (mut psbt, details) = builder.finish().unwrap();
+                    let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    total_sent += 5_000 + details.fee.unwrap_or(0);
+                }
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance after chain");
+
+                // empty wallet
+
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
+
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance empty wallet");
+
+            }
+
+            #[test]
+            fn test_sync_bump_fee_basic() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 5_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance from received");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(2.1));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees after bump");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance from received after bump");
+
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
+            }
+
+            #[test]
+            fn test_sync_bump_fee_remove_change() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect received after send");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(5.0));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after change removal");
+                assert_eq!(new_details.received, 0, "incorrect received after change removal");
+
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
+            }
+
+            #[test]
+            fn test_sync_bump_fee_add_input_simple() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(10.0));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance after add input");
+            }
+
+            #[test]
+            fn test_sync_bump_fee_add_input_no_change() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(123.0));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                println!("{:#?}", new_details);
+
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after add input");
+                assert_eq!(new_details.received, 0, "incorrect received after add input");
+            }
+
+            #[test]
+            fn test_sync_receive_coinbase() {
+                let (wallet, _, mut test_client) = init_single_sig();
+
+                let wallet_addr = wallet.get_address($crate::wallet::AddressIndex::New).unwrap().address;
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance");
+
+                test_client.generate(1, Some(wallet_addr));
+
+                #[cfg(feature = "rpc")]
+                {
+                    // rpc consider coinbase only when mature (100 blocks)
+                    let node_addr = test_client.get_node_address(None);
+                    test_client.generate(100, Some(node_addr));
+                }
+
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert!(wallet.get_balance().unwrap() > 0, "incorrect balance after receiving coinbase");
+            }
+        }
+    };
+
+    ( fn $fn_name:ident ($( $tt:tt )+) -> $blockchain:ty $block:block) => {
+        compile_error!(concat!("Invalid arguments `", stringify!($($tt)*), "` in the blockchain tests fn."));
+        compile_error!("Only the exact `&TestClient` type is supported, **without** any leading path items.");
+    };
+}

src/testutils/mod.rs

@@ -0,0 +1,231 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+#![allow(missing_docs)]
+
+#[cfg(test)]
+#[cfg(feature = "test-blockchains")]
+pub mod blockchain_tests;
+
+use bitcoin::secp256k1::{Secp256k1, Verification};
+use bitcoin::{Address, PublicKey};
+
+use miniscript::descriptor::DescriptorPublicKey;
+use miniscript::{Descriptor, MiniscriptKey, TranslatePk};
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingOutput {
+    pub value: u64,
+    pub to_address: String,
+}
+
+impl TestIncomingOutput {
+    pub fn new(value: u64, to_address: Address) -> Self {
+        Self {
+            value,
+            to_address: to_address.to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingTx {
+    pub output: Vec<TestIncomingOutput>,
+    pub min_confirmations: Option<u64>,
+    pub locktime: Option<i64>,
+    pub replaceable: Option<bool>,
+}
+
+impl TestIncomingTx {
+    pub fn new(
+        output: Vec<TestIncomingOutput>,
+        min_confirmations: Option<u64>,
+        locktime: Option<i64>,
+        replaceable: Option<bool>,
+    ) -> Self {
+        Self {
+            output,
+            min_confirmations,
+            locktime,
+            replaceable,
+        }
+    }
+
+    pub fn add_output(&mut self, output: TestIncomingOutput) {
+        self.output.push(output);
+    }
+}
+
+#[doc(hidden)]
+pub trait TranslateDescriptor {
+    // derive and translate a `Descriptor<DescriptorPublicKey>` into a `Descriptor<PublicKey>`
+    fn derive_translated<C: Verification>(
+        &self,
+        secp: &Secp256k1<C>,
+        index: u32,
+    ) -> Descriptor<PublicKey>;
+}
+
+impl TranslateDescriptor for Descriptor<DescriptorPublicKey> {
+    fn derive_translated<C: Verification>(
+        &self,
+        secp: &Secp256k1<C>,
+        index: u32,
+    ) -> Descriptor<PublicKey> {
+        let translate = |key: &DescriptorPublicKey| -> PublicKey {
+            match key {
+                DescriptorPublicKey::XPub(xpub) => {
+                    xpub.xkey
+                        .derive_pub(secp, &xpub.derivation_path)
+                        .expect("hardened derivation steps")
+                        .public_key
+                }
+                DescriptorPublicKey::SinglePub(key) => key.key,
+            }
+        };
+
+        self.derive(index)
+            .translate_pk_infallible(|pk| translate(pk), |pkh| translate(pkh).to_pubkeyhash())
+    }
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! testutils {
+    ( @external $descriptors:expr, $child:expr ) => ({
+        use $crate::bitcoin::secp256k1::Secp256k1;
+        use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
+
+        use $crate::testutils::TranslateDescriptor;
+
+        let secp = Secp256k1::new();
+
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &$descriptors.0).expect("Failed to parse descriptor in `testutils!(@external)`").0;
+        parsed.derive_translated(&secp, $child).address(bitcoin::Network::Regtest).expect("No address form")
+    });
+    ( @internal $descriptors:expr, $child:expr ) => ({
+        use $crate::bitcoin::secp256k1::Secp256k1;
+        use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
+
+        use $crate::testutils::TranslateDescriptor;
+
+        let secp = Secp256k1::new();
+
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
+        parsed.derive_translated(&secp, $child).address($crate::bitcoin::Network::Regtest).expect("No address form")
+    });
+    ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
+    ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
+
+    ( @tx ( $( ( $( $addr:tt )* ) => $amount:expr ),+ ) $( ( @locktime $locktime:expr ) )? $( ( @confirmations $confirmations:expr ) )? $( ( @replaceable $replaceable:expr ) )? ) => ({
+        let outs = vec![$( $crate::testutils::TestIncomingOutput::new($amount, testutils!( $($addr)* ))),+];
+
+        let locktime = None::<i64>$(.or(Some($locktime)))?;
+
+        let min_confirmations = None::<u64>$(.or(Some($confirmations)))?;
+        let replaceable = None::<bool>$(.or(Some($replaceable)))?;
+
+        $crate::testutils::TestIncomingTx::new(outs, min_confirmations, locktime, replaceable)
+    });
+
+    ( @literal $key:expr ) => ({
+        let key = $key.to_string();
+        (key, None::<String>, None::<String>)
+    });
+    ( @generate_xprv $( $external_path:expr )? $( ,$internal_path:expr )? ) => ({
+        use rand::Rng;
+
+        let mut seed = [0u8; 32];
+        rand::thread_rng().fill(&mut seed[..]);
+
+        let key = $crate::bitcoin::util::bip32::ExtendedPrivKey::new_master(
+            $crate::bitcoin::Network::Testnet,
+            &seed,
+        );
+
+        let external_path = None::<String>$(.or(Some($external_path.to_string())))?;
+        let internal_path = None::<String>$(.or(Some($internal_path.to_string())))?;
+
+        (key.unwrap().to_string(), external_path, internal_path)
+    });
+    ( @generate_wif ) => ({
+        use rand::Rng;
+
+        let mut key = [0u8; $crate::bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
+        rand::thread_rng().fill(&mut key[..]);
+
+        ($crate::bitcoin::PrivateKey {
+            compressed: true,
+            network: $crate::bitcoin::Network::Testnet,
+            key: $crate::bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
+        }.to_string(), None::<String>, None::<String>)
+    });
+
+    ( @keys ( $( $alias:expr => ( $( $key_type:tt )* ) ),+ ) ) => ({
+        let mut map = std::collections::HashMap::new();
+        $(
+            let alias: &str = $alias;
+            map.insert(alias, testutils!( $($key_type)* ));
+        )+
+
+        map
+    });
+
+    ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )? $( ( @keys $( $keys:tt )* ) )* ) => ({
+        use std::str::FromStr;
+        use std::collections::HashMap;
+        use $crate::miniscript::descriptor::Descriptor;
+        use $crate::miniscript::TranslatePk;
+
+        #[allow(unused_assignments, unused_mut)]
+        let mut keys: HashMap<&'static str, (String, Option<String>, Option<String>)> = HashMap::new();
+        $(
+            keys = testutils!{ @keys $( $keys )* };
+        )*
+
+        let external: Descriptor<String> = FromStr::from_str($external_descriptor).unwrap();
+        let external: Descriptor<String> = external.translate_pk_infallible::<_, _>(|k| {
+            if let Some((key, ext_path, _)) = keys.get(&k.as_str()) {
+                format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))
+            } else {
+                k.clone()
+            }
+        }, |kh| {
+            if let Some((key, ext_path, _)) = keys.get(&kh.as_str()) {
+                format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))
+            } else {
+                kh.clone()
+            }
+
+        });
+        let external = external.to_string();
+
+        let internal = None::<String>$(.or({
+            let string_internal: Descriptor<String> = FromStr::from_str($internal_descriptor).unwrap();
+
+            let string_internal: Descriptor<String> = string_internal.translate_pk_infallible::<_, _>(|k| {
+                if let Some((key, _, int_path)) = keys.get(&k.as_str()) {
+                    format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))
+                } else {
+                    k.clone()
+                }
+            }, |kh| {
+                if let Some((key, _, int_path)) = keys.get(&kh.as_str()) {
+                    format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))
+                } else {
+                    kh.clone()
+                }
+            });
+            Some(string_internal.to_string())
+        }))?;
+
+        (external, internal)
+    })
+}

src/types.rs

@@ -1,31 +1,19 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::convert::AsRef;
+use std::ops::Sub;
 
 use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
-use bitcoin::hash_types::Txid;
+use bitcoin::{hash_types::Txid, util::psbt};
 
 use serde::{Deserialize, Serialize};
 
@@ -69,19 +57,40 @@ impl FeeRate {
     }
 
     /// Create a new instance of [`FeeRate`] given a float fee rate in satoshi/vbyte
-    pub fn from_sat_per_vb(sat_per_vb: f32) -> Self {
+    pub const fn from_sat_per_vb(sat_per_vb: f32) -> Self {
         FeeRate(sat_per_vb)
     }
 
     /// Create a new [`FeeRate`] with the default min relay fee value
-    pub fn default_min_relay_fee() -> Self {
+    pub const fn default_min_relay_fee() -> Self {
         FeeRate(1.0)
     }
 
+    /// Calculate fee rate from `fee` and weight units (`wu`).
+    pub fn from_wu(fee: u64, wu: usize) -> FeeRate {
+        Self::from_vb(fee, wu.vbytes())
+    }
+
+    /// Calculate fee rate from `fee` and `vbytes`.
+    pub fn from_vb(fee: u64, vbytes: usize) -> FeeRate {
+        let rate = fee as f32 / vbytes as f32;
+        Self::from_sat_per_vb(rate)
+    }
+
     /// Return the value as satoshi/vbyte
     pub fn as_sat_vb(&self) -> f32 {
         self.0
     }
+
+    /// Calculate absolute fee in Satoshis using size in weight units.
+    pub fn fee_wu(&self, wu: usize) -> u64 {
+        self.fee_vb(wu.vbytes())
+    }
+
+    /// Calculate absolute fee in Satoshis using size in virtual bytes.
+    pub fn fee_vb(&self, vbytes: usize) -> u64 {
+        (self.as_sat_vb() * vbytes as f32).ceil() as u64
+    }
 }
 
 impl std::default::Default for FeeRate {
@@ -90,9 +99,32 @@ impl std::default::Default for FeeRate {
     }
 }
 
-/// A wallet unspent output
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub struct UTXO {
+impl Sub for FeeRate {
+    type Output = Self;
+
+    fn sub(self, other: FeeRate) -> Self::Output {
+        FeeRate(self.0 - other.0)
+    }
+}
+
+/// Trait implemented by types that can be used to measure weight units.
+pub trait Vbytes {
+    /// Convert weight units to virtual bytes.
+    fn vbytes(self) -> usize;
+}
+
+impl Vbytes for usize {
+    fn vbytes(self) -> usize {
+        // ref: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki#transaction-size-calculations
+        (self as f32 / 4.0).ceil() as usize
+    }
+}
+
+/// An unspent output owned by a [`Wallet`].
+///
+/// [`Wallet`]: crate::Wallet
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LocalUtxo {
     /// Reference to a transaction output
     pub outpoint: OutPoint,
     /// Transaction output
@@ -101,6 +133,64 @@ pub struct UTXO {
     pub keychain: KeychainKind,
 }
 
+/// A [`Utxo`] with its `satisfaction_weight`.
+#[derive(Debug, Clone, PartialEq)]
+pub struct WeightedUtxo {
+    /// The weight of the witness data and `scriptSig` expressed in [weight units]. This is used to
+    /// properly maintain the feerate when adding this input to a transaction during coin selection.
+    ///
+    /// [weight units]: https://en.bitcoin.it/wiki/Weight_units
+    pub satisfaction_weight: usize,
+    /// The UTXO
+    pub utxo: Utxo,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+/// An unspent transaction output (UTXO).
+pub enum Utxo {
+    /// A UTXO owned by the local wallet.
+    Local(LocalUtxo),
+    /// A UTXO owned by another wallet.
+    Foreign {
+        /// The location of the output.
+        outpoint: OutPoint,
+        /// The information about the input we require to add it to a PSBT.
+        // Box it to stop the type being too big.
+        psbt_input: Box<psbt::Input>,
+    },
+}
+
+impl Utxo {
+    /// Get the location of the UTXO
+    pub fn outpoint(&self) -> OutPoint {
+        match &self {
+            Utxo::Local(local) => local.outpoint,
+            Utxo::Foreign { outpoint, .. } => *outpoint,
+        }
+    }
+
+    /// Get the `TxOut` of the UTXO
+    pub fn txout(&self) -> &TxOut {
+        match &self {
+            Utxo::Local(local) => &local.txout,
+            Utxo::Foreign {
+                outpoint,
+                psbt_input,
+            } => {
+                if let Some(prev_tx) = &psbt_input.non_witness_utxo {
+                    return &prev_tx.output[outpoint.vout as usize];
+                }
+
+                if let Some(txout) = &psbt_input.witness_utxo {
+                    return txout;
+                }
+
+                unreachable!("Foreign UTXOs will always have one of these set")
+            }
+        }
+    }
+}
+
 /// A wallet transaction
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
 pub struct TransactionDetails {
@@ -108,14 +198,56 @@ pub struct TransactionDetails {
     pub transaction: Option<Transaction>,
     /// Transaction id
     pub txid: Txid,
-    /// Timestamp
-    pub timestamp: u64,
+
     /// Received value (sats)
     pub received: u64,
     /// Sent value (sats)
     pub sent: u64,
-    /// Fee value (sats)
-    pub fees: u64,
-    /// Confirmed in block height, `None` means unconfirmed
-    pub height: Option<u32>,
+    /// Fee value (sats) if available.
+    /// The availability of the fee depends on the backend. It's never `None` with an Electrum
+    /// Server backend, but it could be `None` with a Bitcoin RPC node without txindex that receive
+    /// funds while offline.
+    pub fee: Option<u64>,
+    /// If the transaction is confirmed, contains height and timestamp of the block containing the
+    /// transaction, unconfirmed transaction contains `None`.
+    pub confirmation_time: Option<ConfirmationTime>,
+    /// Whether the tx has been verified against the consensus rules
+    ///
+    /// Confirmed txs are considered "verified" by default, while unconfirmed txs are checked to
+    /// ensure an unstrusted [`Blockchain`](crate::blockchain::Blockchain) backend can't trick the
+    /// wallet into using an invalid tx as an RBF template.
+    ///
+    /// The check is only perfomed when the `verify` feature is enabled.
+    #[serde(default = "bool::default")] // default to `false` if not specified
+    pub verified: bool,
+}
+
+/// Block height and timestamp of the block containing the confirmed transaction
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
+pub struct ConfirmationTime {
+    /// confirmation block height
+    pub height: u32,
+    /// confirmation block timestamp
+    pub timestamp: u64,
+}
+
+impl ConfirmationTime {
+    /// Returns `Some` `ConfirmationTime` if both `height` and `timestamp` are `Some`
+    pub fn new(height: Option<u32>, timestamp: Option<u64>) -> Option<Self> {
+        match (height, timestamp) {
+            (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn can_store_feerate_in_const() {
+        const _MY_RATE: FeeRate = FeeRate::from_sat_per_vb(10.0);
+        const _MIN_RELAY: FeeRate = FeeRate::default_min_relay_fee();
+    }
 }

src/wallet/address_validator.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Address validation callbacks
 //!
@@ -33,7 +20,7 @@
 //! An address validator can be attached to a [`Wallet`](super::Wallet) by using the
 //! [`Wallet::add_address_validator`](super::Wallet::add_address_validator) method, and
 //! whenever a new address is generated (either explicitly by the user with
-//! [`Wallet::get_new_address`](super::Wallet::get_new_address) or internally to create a change
+//! [`Wallet::get_address`](super::Wallet::get_address) or internally to create a change
 //! address) all the attached validators will be polled, in sequence. All of them must complete
 //! successfully to continue.
 //!
@@ -45,6 +32,7 @@
 //! # use bdk::address_validator::*;
 //! # use bdk::database::*;
 //! # use bdk::*;
+//! # use bdk::wallet::AddressIndex::New;
 //! #[derive(Debug)]
 //! struct PrintAddressAndContinue;
 //!
@@ -52,7 +40,7 @@
 //!     fn validate(
 //!         &self,
 //!         keychain: KeychainKind,
-//!         hd_keypaths: &HDKeyPaths,
+//!         hd_keypaths: &HdKeyPaths,
 //!         script: &Script
 //!     ) -> Result<(), AddressValidatorError> {
 //!         let address = Address::from_script(script, Network::Testnet)
@@ -70,7 +58,7 @@
 //! let mut wallet = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
 //! wallet.add_address_validator(Arc::new(PrintAddressAndContinue));
 //!
-//! let address = wallet.get_new_address()?;
+//! let address = wallet.get_address(New)?;
 //! println!("Address: {}", address);
 //! # Ok::<(), bdk::Error>(())
 //! ```
@@ -79,7 +67,7 @@ use std::fmt;
 
 use bitcoin::Script;
 
-use crate::descriptor::HDKeyPaths;
+use crate::descriptor::HdKeyPaths;
 use crate::types::KeychainKind;
 
 /// Errors that can be returned to fail the validation of an address
@@ -117,7 +105,7 @@ pub trait AddressValidator: Send + Sync + fmt::Debug {
     fn validate(
         &self,
         keychain: KeychainKind,
-        hd_keypaths: &HDKeyPaths,
+        hd_keypaths: &HdKeyPaths,
         script: &Script,
     ) -> Result<(), AddressValidatorError>;
 }
@@ -127,7 +115,8 @@ mod test {
     use std::sync::Arc;
 
     use super::*;
-    use crate::wallet::test::{get_funded_wallet, get_test_wpkh};
+    use crate::wallet::AddressIndex::New;
+    use crate::wallet::{get_funded_wallet, test::get_test_wpkh};
 
     #[derive(Debug)]
     struct TestValidator;
@@ -135,7 +124,7 @@ mod test {
         fn validate(
             &self,
             _keychain: KeychainKind,
-            _hd_keypaths: &HDKeyPaths,
+            _hd_keypaths: &HdKeyPaths,
             _script: &bitcoin::Script,
         ) -> Result<(), AddressValidatorError> {
             Err(AddressValidatorError::InvalidScript)
@@ -148,7 +137,7 @@ mod test {
         let (mut wallet, _, _) = get_funded_wallet(get_test_wpkh());
         wallet.add_address_validator(Arc::new(TestValidator));
 
-        wallet.get_new_address().unwrap();
+        wallet.get_address(New).unwrap();
     }
 
     #[test]
@@ -157,7 +146,7 @@ mod test {
         let (mut wallet, descriptors, _) = get_funded_wallet(get_test_wpkh());
         wallet.add_address_validator(Arc::new(TestValidator));
 
-        let addr = testutils!(@external descriptors, 10);
+        let addr = crate::testutils!(@external descriptors, 10);
         let mut builder = wallet.build_tx();
         builder.add_recipient(addr.script_pubkey(), 25_000);
         builder.finish().unwrap();

src/wallet/coin_selection.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Coin selection
 //!
@@ -39,7 +26,7 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::wallet::coin_selection::*;
+//! # use bdk::wallet::{self, coin_selection::*};
 //! # use bdk::database::Database;
 //! # use bdk::*;
 //! # const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4 + 1) * 4;
@@ -50,32 +37,37 @@
 //!     fn coin_select(
 //!         &self,
 //!         database: &D,
-//!         required_utxos: Vec<(UTXO, usize)>,
-//!         optional_utxos: Vec<(UTXO, usize)>,
+//!         required_utxos: Vec<WeightedUtxo>,
+//!         optional_utxos: Vec<WeightedUtxo>,
 //!         fee_rate: FeeRate,
 //!         amount_needed: u64,
-//!         fee_amount: f32,
+//!         fee_amount: u64,
 //!     ) -> Result<CoinSelectionResult, bdk::Error> {
 //!         let mut selected_amount = 0;
 //!         let mut additional_weight = 0;
 //!         let all_utxos_selected = required_utxos
-//!             .into_iter().chain(optional_utxos)
-//!             .scan((&mut selected_amount, &mut additional_weight), |(selected_amount, additional_weight), (utxo, weight)| {
-//!                 **selected_amount += utxo.txout.value;
-//!                 **additional_weight += TXIN_BASE_WEIGHT + weight;
-//!
-//!                 Some(utxo)
-//!             })
+//!             .into_iter()
+//!             .chain(optional_utxos)
+//!             .scan(
+//!                 (&mut selected_amount, &mut additional_weight),
+//!                 |(selected_amount, additional_weight), weighted_utxo| {
+//!                     **selected_amount += weighted_utxo.utxo.txout().value;
+//!                     **additional_weight += TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight;
+//!                     Some(weighted_utxo.utxo)
+//!                 },
+//!             )
 //!             .collect::<Vec<_>>();
-//!         let additional_fees = additional_weight as f32 * fee_rate.as_sat_vb() / 4.0;
-//!         let amount_needed_with_fees = (fee_amount + additional_fees).ceil() as u64 + amount_needed;
-//!         if  amount_needed_with_fees > selected_amount {
-//!             return Err(bdk::Error::InsufficientFunds{ needed: amount_needed_with_fees, available: selected_amount });
+//!         let additional_fees = fee_rate.fee_wu(additional_weight);
+//!         let amount_needed_with_fees = (fee_amount + additional_fees) + amount_needed;
+//!         if amount_needed_with_fees > selected_amount {
+//!             return Err(bdk::Error::InsufficientFunds {
+//!                 needed: amount_needed_with_fees,
+//!                 available: selected_amount,
+//!             });
 //!         }
 //!
 //!         Ok(CoinSelectionResult {
 //!             selected: all_utxos_selected,
-//!             selected_amount,
 //!             fee_amount: fee_amount + additional_fees,
 //!         })
 //!     }
@@ -87,8 +79,7 @@
 //! let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
 //! let (psbt, details) = {
 //!     let mut builder = wallet.build_tx().coin_selection(AlwaysSpendEverything);
-//!     builder
-//!         .add_recipient(to_address.script_pubkey(), 50_000);
+//!     builder.add_recipient(to_address.script_pubkey(), 50_000);
 //!     builder.finish()?
 //! };
 //!
@@ -97,15 +88,16 @@
 //! # Ok::<(), bdk::Error>(())
 //! ```
 
-use crate::database::Database;
-use crate::error::Error;
-use crate::types::{FeeRate, UTXO};
+use crate::types::FeeRate;
+use crate::{database::Database, WeightedUtxo};
+use crate::{error::Error, Utxo};
 
 use rand::seq::SliceRandom;
 #[cfg(not(test))]
 use rand::thread_rng;
 #[cfg(test)]
 use rand::{rngs::StdRng, SeedableRng};
+use std::convert::TryInto;
 
 /// Default coin selection algorithm used by [`TxBuilder`](super::tx_builder::TxBuilder) if not
 /// overridden
@@ -122,11 +114,27 @@ pub(crate) const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4 + 1) * 4;
 #[derive(Debug)]
 pub struct CoinSelectionResult {
     /// List of outputs selected for use as inputs
-    pub selected: Vec<UTXO>,
-    /// Sum of the selected inputs' value
-    pub selected_amount: u64,
+    pub selected: Vec<Utxo>,
     /// Total fee amount in satoshi
-    pub fee_amount: f32,
+    pub fee_amount: u64,
+}
+
+impl CoinSelectionResult {
+    /// The total value of the inputs selected.
+    pub fn selected_amount(&self) -> u64 {
+        self.selected.iter().map(|u| u.txout().value).sum()
+    }
+
+    /// The total value of the inputs selected from the local wallet.
+    pub fn local_selected_amount(&self) -> u64 {
+        self.selected
+            .iter()
+            .filter_map(|u| match u {
+                Utxo::Local(_) => Some(u.txout().value),
+                _ => None,
+            })
+            .sum()
+    }
 }
 
 /// Trait for generalized coin selection algorithms
@@ -151,11 +159,11 @@ pub trait CoinSelectionAlgorithm<D: Database>: std::fmt::Debug {
     fn coin_select(
         &self,
         database: &D,
-        required_utxos: Vec<(UTXO, usize)>,
-        optional_utxos: Vec<(UTXO, usize)>,
+        required_utxos: Vec<WeightedUtxo>,
+        optional_utxos: Vec<WeightedUtxo>,
         fee_rate: FeeRate,
         amount_needed: u64,
-        fee_amount: f32,
+        fee_amount: u64,
     ) -> Result<CoinSelectionResult, Error>;
 }
 
@@ -163,21 +171,19 @@ pub trait CoinSelectionAlgorithm<D: Database>: std::fmt::Debug {
 ///
 /// This coin selection algorithm sorts the available UTXOs by value and then picks them starting
 /// from the largest ones until the required amount is reached.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone, Copy)]
 pub struct LargestFirstCoinSelection;
 
 impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
     fn coin_select(
         &self,
         _database: &D,
-        required_utxos: Vec<(UTXO, usize)>,
-        mut optional_utxos: Vec<(UTXO, usize)>,
+        required_utxos: Vec<WeightedUtxo>,
+        mut optional_utxos: Vec<WeightedUtxo>,
         fee_rate: FeeRate,
         amount_needed: u64,
-        mut fee_amount: f32,
+        mut fee_amount: u64,
     ) -> Result<CoinSelectionResult, Error> {
-        let calc_fee_bytes = |wu| (wu as f32) * fee_rate.as_sat_vb() / 4.0;
-
         log::debug!(
             "amount_needed = `{}`, fee_amount = `{}`, fee_rate = `{:?}`",
             amount_needed,
@@ -188,7 +194,7 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
         // We put the "required UTXOs" first and make sure the optional UTXOs are sorted,
         // initially smallest to largest, before being reversed with `.rev()`.
         let utxos = {
-            optional_utxos.sort_unstable_by_key(|(utxo, _)| utxo.txout.value);
+            optional_utxos.sort_unstable_by_key(|wu| wu.utxo.txout().value);
             required_utxos
                 .into_iter()
                 .map(|utxo| (true, utxo))
@@ -201,18 +207,19 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
         let selected = utxos
             .scan(
                 (&mut selected_amount, &mut fee_amount),
-                |(selected_amount, fee_amount), (must_use, (utxo, weight))| {
-                    if must_use || **selected_amount < amount_needed + (fee_amount.ceil() as u64) {
-                        **fee_amount += calc_fee_bytes(TXIN_BASE_WEIGHT + weight);
-                        **selected_amount += utxo.txout.value;
+                |(selected_amount, fee_amount), (must_use, weighted_utxo)| {
+                    if must_use || **selected_amount < amount_needed + **fee_amount {
+                        **fee_amount +=
+                            fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
+                        **selected_amount += weighted_utxo.utxo.txout().value;
 
                         log::debug!(
                             "Selected {}, updated fee_amount = `{}`",
-                            utxo.outpoint,
+                            weighted_utxo.utxo.outpoint(),
                             fee_amount
                         );
 
-                        Some(utxo)
+                        Some(weighted_utxo.utxo)
                     } else {
                         None
                     }
@@ -220,7 +227,7 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
             )
             .collect::<Vec<_>>();
 
-        let amount_needed_with_fees = amount_needed + (fee_amount.ceil() as u64);
+        let amount_needed_with_fees = amount_needed + fee_amount;
         if selected_amount < amount_needed_with_fees {
             return Err(Error::InsufficientFunds {
                 needed: amount_needed_with_fees,
@@ -231,7 +238,6 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
         Ok(CoinSelectionResult {
             selected,
             fee_amount,
-            selected_amount,
         })
     }
 }
@@ -239,24 +245,21 @@ impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
 #[derive(Debug, Clone)]
 // Adds fee information to an UTXO.
 struct OutputGroup {
-    utxo: UTXO,
-    // weight needed to satisfy the UTXO, as described in `Descriptor::max_satisfaction_weight`
-    satisfaction_weight: usize,
+    weighted_utxo: WeightedUtxo,
     // Amount of fees for spending a certain utxo, calculated using a certain FeeRate
-    fee: f32,
+    fee: u64,
     // The effective value of the UTXO, i.e., the utxo value minus the fee for spending it
     effective_value: i64,
 }
 
 impl OutputGroup {
-    fn new(utxo: UTXO, satisfaction_weight: usize, fee_rate: FeeRate) -> Self {
-        let fee = (TXIN_BASE_WEIGHT + satisfaction_weight) as f32 / 4.0 * fee_rate.as_sat_vb();
-        let effective_value = utxo.txout.value as i64 - fee.ceil() as i64;
+    fn new(weighted_utxo: WeightedUtxo, fee_rate: FeeRate) -> Self {
+        let fee = fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
+        let effective_value = weighted_utxo.utxo.txout().value as i64 - fee as i64;
         OutputGroup {
-            utxo,
-            satisfaction_weight,
-            effective_value,
+            weighted_utxo,
             fee,
+            effective_value,
         }
     }
 }
@@ -291,45 +294,60 @@ impl<D: Database> CoinSelectionAlgorithm<D> for BranchAndBoundCoinSelection {
     fn coin_select(
         &self,
         _database: &D,
-        required_utxos: Vec<(UTXO, usize)>,
-        optional_utxos: Vec<(UTXO, usize)>,
+        required_utxos: Vec<WeightedUtxo>,
+        optional_utxos: Vec<WeightedUtxo>,
         fee_rate: FeeRate,
         amount_needed: u64,
-        fee_amount: f32,
+        fee_amount: u64,
     ) -> Result<CoinSelectionResult, Error> {
         // Mapping every (UTXO, usize) to an output group
         let required_utxos: Vec<OutputGroup> = required_utxos
             .into_iter()
-            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
         // Mapping every (UTXO, usize) to an output group.
-        // Filtering UTXOs with an effective_value < 0, as the fee paid for
-        // adding them is more than their value
         let optional_utxos: Vec<OutputGroup> = optional_utxos
             .into_iter()
-            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
-            .filter(|u| u.effective_value > 0)
+            .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
         let curr_value = required_utxos
             .iter()
-            .fold(0, |acc, x| acc + x.effective_value as u64);
+            .fold(0, |acc, x| acc + x.effective_value);
 
         let curr_available_value = optional_utxos
             .iter()
-            .fold(0, |acc, x| acc + x.effective_value as u64);
+            .fold(0, |acc, x| acc + x.effective_value);
 
-        let actual_target = fee_amount.ceil() as u64 + amount_needed;
+        let actual_target = fee_amount + amount_needed;
         let cost_of_change = self.size_of_change as f32 * fee_rate.as_sat_vb();
 
-        if curr_available_value + curr_value < actual_target {
+        let expected = (curr_available_value + curr_value)
+            .try_into()
+            .map_err(|_| {
+                Error::Generic("Sum of UTXO spendable values does not fit into u64".to_string())
+            })?;
+
+        if expected < actual_target {
             return Err(Error::InsufficientFunds {
                 needed: actual_target,
-                available: curr_available_value + curr_value,
+                available: expected,
             });
         }
 
+        let actual_target = actual_target
+            .try_into()
+            .expect("Bitcoin amount to fit into i64");
+
+        if curr_value > actual_target {
+            return Ok(BranchAndBoundCoinSelection::calculate_cs_result(
+                vec![],
+                required_utxos,
+                fee_amount,
+            ));
+        }
+
         Ok(self
             .bnb(
                 required_utxos.clone(),
@@ -360,10 +378,10 @@ impl BranchAndBoundCoinSelection {
         &self,
         required_utxos: Vec<OutputGroup>,
         mut optional_utxos: Vec<OutputGroup>,
-        mut curr_value: u64,
-        mut curr_available_value: u64,
-        actual_target: u64,
-        fee_amount: f32,
+        mut curr_value: i64,
+        mut curr_available_value: i64,
+        actual_target: i64,
+        fee_amount: u64,
         cost_of_change: f32,
     ) -> Result<CoinSelectionResult, Error> {
         // current_selection[i] will contain true if we are using optional_utxos[i],
@@ -388,7 +406,7 @@ impl BranchAndBoundCoinSelection {
             // or the selected value is out of range.
             // Go back and try other branch
             if curr_value + curr_available_value < actual_target
-                || curr_value > actual_target + cost_of_change as u64
+                || curr_value > actual_target + cost_of_change as i64
             {
                 backtrack = true;
             } else if curr_value >= actual_target {
@@ -414,8 +432,7 @@ impl BranchAndBoundCoinSelection {
                 // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed.
                 while let Some(false) = current_selection.last() {
                     current_selection.pop();
-                    curr_available_value +=
-                        optional_utxos[current_selection.len()].effective_value as u64;
+                    curr_available_value += optional_utxos[current_selection.len()].effective_value;
                 }
 
                 if current_selection.last_mut().is_none() {
@@ -433,17 +450,17 @@ impl BranchAndBoundCoinSelection {
                 }
 
                 let utxo = &optional_utxos[current_selection.len() - 1];
-                curr_value -= utxo.effective_value as u64;
+                curr_value -= utxo.effective_value;
             } else {
                 // Moving forwards, continuing down this branch
                 let utxo = &optional_utxos[current_selection.len()];
 
                 // Remove this utxo from the curr_available_value utxo amount
-                curr_available_value -= utxo.effective_value as u64;
+                curr_available_value -= utxo.effective_value;
 
                 // Inclusion branch first (Largest First Exploration)
                 current_selection.push(true);
-                curr_value += utxo.effective_value as u64;
+                curr_value += utxo.effective_value;
             }
         }
 
@@ -470,9 +487,9 @@ impl BranchAndBoundCoinSelection {
         &self,
         required_utxos: Vec<OutputGroup>,
         mut optional_utxos: Vec<OutputGroup>,
-        curr_value: u64,
-        actual_target: u64,
-        fee_amount: f32,
+        curr_value: i64,
+        actual_target: i64,
+        fee_amount: u64,
     ) -> CoinSelectionResult {
         #[cfg(not(test))]
         optional_utxos.shuffle(&mut thread_rng());
@@ -489,7 +506,7 @@ impl BranchAndBoundCoinSelection {
                 if *curr_value >= actual_target {
                     None
                 } else {
-                    *curr_value += utxo.effective_value as u64;
+                    *curr_value += utxo.effective_value;
                     Some(utxo)
                 }
             })
@@ -501,20 +518,18 @@ impl BranchAndBoundCoinSelection {
     fn calculate_cs_result(
         mut selected_utxos: Vec<OutputGroup>,
         mut required_utxos: Vec<OutputGroup>,
-        mut fee_amount: f32,
+        mut fee_amount: u64,
     ) -> CoinSelectionResult {
         selected_utxos.append(&mut required_utxos);
-        fee_amount += selected_utxos.iter().map(|u| u.fee).sum::<f32>();
+        fee_amount += selected_utxos.iter().map(|u| u.fee).sum::<u64>();
         let selected = selected_utxos
             .into_iter()
-            .map(|u| u.utxo)
+            .map(|u| u.weighted_utxo.utxo)
             .collect::<Vec<_>>();
-        let selected_amount = selected.iter().map(|u| u.txout.value).sum();
 
         CoinSelectionResult {
             selected,
             fee_amount,
-            selected_amount,
         }
     }
 }
@@ -528,6 +543,7 @@ mod test {
     use super::*;
     use crate::database::MemoryDatabase;
     use crate::types::*;
+    use crate::wallet::Vbytes;
 
     use rand::rngs::StdRng;
     use rand::seq::SliceRandom;
@@ -535,44 +551,42 @@ mod test {
 
     const P2WPKH_WITNESS_SIZE: usize = 73 + 33 + 2;
 
-    fn get_test_utxos() -> Vec<(UTXO, usize)> {
+    const FEE_AMOUNT: u64 = 50;
+
+    fn utxo(value: u64, index: u32) -> WeightedUtxo {
+        assert!(index < 10);
+        let outpoint = OutPoint::from_str(&format!(
+            "000000000000000000000000000000000000000000000000000000000000000{}:0",
+            index
+        ))
+        .unwrap();
+        WeightedUtxo {
+            satisfaction_weight: P2WPKH_WITNESS_SIZE,
+            utxo: Utxo::Local(LocalUtxo {
+                outpoint,
+                txout: TxOut {
+                    value,
+                    script_pubkey: Script::new(),
+                },
+                keychain: KeychainKind::External,
+            }),
+        }
+    }
+
+    fn get_test_utxos() -> Vec<WeightedUtxo> {
         vec![
-            (
-                UTXO {
-                    outpoint: OutPoint::from_str(
-                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
-                    )
-                    .unwrap(),
-                    txout: TxOut {
-                        value: 100_000,
-                        script_pubkey: Script::new(),
-                    },
-                    keychain: KeychainKind::External,
-                },
-                P2WPKH_WITNESS_SIZE,
-            ),
-            (
-                UTXO {
-                    outpoint: OutPoint::from_str(
-                        "65d92ddff6b6dc72c89624a6491997714b90f6004f928d875bc0fd53f264fa85:0",
-                    )
-                    .unwrap(),
-                    txout: TxOut {
-                        value: 200_000,
-                        script_pubkey: Script::new(),
-                    },
-                    keychain: KeychainKind::Internal,
-                },
-                P2WPKH_WITNESS_SIZE,
-            ),
+            utxo(100_000, 0),
+            utxo(FEE_AMOUNT as u64 - 40, 1),
+            utxo(200_000, 2),
         ]
     }
 
-    fn generate_random_utxos(rng: &mut StdRng, utxos_number: usize) -> Vec<(UTXO, usize)> {
+    fn generate_random_utxos(rng: &mut StdRng, utxos_number: usize) -> Vec<WeightedUtxo> {
         let mut res = Vec::new();
         for _ in 0..utxos_number {
-            res.push((
-                UTXO {
+            res.push(WeightedUtxo {
+                satisfaction_weight: P2WPKH_WITNESS_SIZE,
+                utxo: Utxo::Local(LocalUtxo {
                     outpoint: OutPoint::from_str(
                         "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
                     )
@@ -582,16 +596,16 @@ mod test {
                         script_pubkey: Script::new(),
                     },
                     keychain: KeychainKind::External,
-                },
-                P2WPKH_WITNESS_SIZE,
-            ));
+                }),
+            });
         }
         res
     }
 
-    fn generate_same_value_utxos(utxos_value: u64, utxos_number: usize) -> Vec<(UTXO, usize)> {
-        let utxo = (
-            UTXO {
+    fn generate_same_value_utxos(utxos_value: u64, utxos_number: usize) -> Vec<WeightedUtxo> {
+        let utxo = WeightedUtxo {
+            satisfaction_weight: P2WPKH_WITNESS_SIZE,
+            utxo: Utxo::Local(LocalUtxo {
                 outpoint: OutPoint::from_str(
                     "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
                 )
@@ -601,18 +615,18 @@ mod test {
                     script_pubkey: Script::new(),
                 },
                 keychain: KeychainKind::External,
-            },
-            P2WPKH_WITNESS_SIZE,
-        );
+            }),
+        };
         vec![utxo; utxos_number]
     }
 
-    fn sum_random_utxos(mut rng: &mut StdRng, utxos: &mut Vec<(UTXO, usize)>) -> u64 {
+    fn sum_random_utxos(mut rng: &mut StdRng, utxos: &mut Vec<WeightedUtxo>) -> u64 {
         let utxos_picked_len = rng.gen_range(2, utxos.len() / 2);
         utxos.shuffle(&mut rng);
         utxos[..utxos_picked_len]
             .iter()
-            .fold(0, |acc, x| acc + x.0.txout.value)
+            .map(|u| u.utxo.txout().value)
+            .sum()
     }
 
     #[test]
@@ -627,13 +641,13 @@ mod test {
                 vec![],
                 FeeRate::from_sat_per_vb(1.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
-        assert_eq!(result.selected.len(), 2);
-        assert_eq!(result.selected_amount, 300_000);
-        assert_eq!(result.fee_amount, 186.0);
+        assert_eq!(result.selected.len(), 3);
+        assert_eq!(result.selected_amount(), 300_010);
+        assert_eq!(result.fee_amount, 254)
     }
 
     #[test]
@@ -648,13 +662,13 @@ mod test {
                 vec![],
                 FeeRate::from_sat_per_vb(1.0),
                 20_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
-        assert_eq!(result.selected.len(), 2);
-        assert_eq!(result.selected_amount, 300_000);
-        assert_eq!(result.fee_amount, 186.0);
+        assert_eq!(result.selected.len(), 3);
+        assert_eq!(result.selected_amount(), 300_010);
+        assert_eq!(result.fee_amount, 254);
     }
 
     #[test]
@@ -669,13 +683,13 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 20_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 1);
-        assert_eq!(result.selected_amount, 200_000);
-        assert_eq!(result.fee_amount, 118.0);
+        assert_eq!(result.selected_amount(), 200_000);
+        assert_eq!(result.fee_amount, 118);
     }
 
     #[test]
@@ -691,7 +705,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 500_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -709,7 +723,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1000.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -729,13 +743,13 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 3);
-        assert_eq!(result.selected_amount, 300_000);
-        assert_eq!(result.fee_amount, 254.0);
+        assert_eq!(result.selected_amount(), 300_000);
+        assert_eq!(result.fee_amount, 254);
     }
 
     #[test]
@@ -750,13 +764,65 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 20_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
 
-        assert_eq!(result.selected.len(), 2);
-        assert_eq!(result.selected_amount, 300_000);
-        assert_eq!(result.fee_amount, 186.0);
+        assert_eq!(result.selected.len(), 3);
+        assert_eq!(result.selected_amount(), 300_010);
+        assert_eq!(result.fee_amount, 254);
+    }
+
+    #[test]
+    fn test_bnb_coin_selection_optional_are_enough() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let result = BranchAndBoundCoinSelection::default()
+            .coin_select(
+                &database,
+                vec![],
+                utxos,
+                FeeRate::from_sat_per_vb(1.0),
+                299756,
+                FEE_AMOUNT,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 3);
+        assert_eq!(result.selected_amount(), 300010);
+        assert_eq!(result.fee_amount, 254);
+    }
+
+    #[test]
+    fn test_bnb_coin_selection_required_not_enough() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let required = vec![utxos[0].clone()];
+        let mut optional = utxos[1..].to_vec();
+        optional.push(utxo(500_000, 3));
+
+        // Defensive assertions, for sanity and in case someone changes the test utxos vector.
+        let amount: u64 = required.iter().map(|u| u.utxo.txout().value).sum();
+        assert_eq!(amount, 100_000);
+        let amount: u64 = optional.iter().map(|u| u.utxo.txout().value).sum();
+        assert!(amount > 150_000);
+
+        let result = BranchAndBoundCoinSelection::default()
+            .coin_select(
+                &database,
+                required,
+                optional,
+                FeeRate::from_sat_per_vb(1.0),
+                150_000,
+                FEE_AMOUNT,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 3);
+        assert_eq!(result.selected_amount(), 300_010);
+        assert!((result.fee_amount as f32 - 254.0).abs() < f32::EPSILON);
     }
 
     #[test]
@@ -772,7 +838,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 500_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -790,7 +856,7 @@ mod test {
                 utxos,
                 FeeRate::from_sat_per_vb(1000.0),
                 250_000,
-                50.0,
+                FEE_AMOUNT,
             )
             .unwrap();
     }
@@ -804,18 +870,18 @@ mod test {
             .coin_select(
                 &database,
                 vec![],
-                utxos.clone(),
+                utxos,
                 FeeRate::from_sat_per_vb(1.0),
                 99932, // first utxo's effective value
-                0.0,
+                0,
             )
             .unwrap();
 
         assert_eq!(result.selected.len(), 1);
-        assert_eq!(result.selected_amount, 100_000);
-        let input_size = (TXIN_BASE_WEIGHT as f32) / 4.0 + P2WPKH_WITNESS_SIZE as f32 / 4.0;
+        assert_eq!(result.selected_amount(), 100_000);
+        let input_size = (TXIN_BASE_WEIGHT + P2WPKH_WITNESS_SIZE).vbytes();
         let epsilon = 0.5;
-        assert!((1.0 - (result.fee_amount / input_size)).abs() < epsilon);
+        assert!((1.0 - (result.fee_amount as f32 / input_size as f32)).abs() < epsilon);
     }
 
     #[test]
@@ -834,10 +900,10 @@ mod test {
                     optional_utxos,
                     FeeRate::from_sat_per_vb(0.0),
                     target_amount,
-                    0.0,
+                    0,
                 )
                 .unwrap();
-            assert_eq!(result.selected_amount, target_amount);
+            assert_eq!(result.selected_amount(), target_amount);
         }
     }
 
@@ -847,12 +913,10 @@ mod test {
         let fee_rate = FeeRate::from_sat_per_vb(10.0);
         let utxos: Vec<OutputGroup> = get_test_utxos()
             .into_iter()
-            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
-        let curr_available_value = utxos
-            .iter()
-            .fold(0, |acc, x| acc + x.effective_value as u64);
+        let curr_available_value = utxos.iter().fold(0, |acc, x| acc + x.effective_value);
 
         let size_of_change = 31;
         let cost_of_change = size_of_change as f32 * fee_rate.as_sat_vb();
@@ -863,7 +927,7 @@ mod test {
                 0,
                 curr_available_value,
                 20_000,
-                50.0,
+                FEE_AMOUNT,
                 cost_of_change,
             )
             .unwrap();
@@ -875,12 +939,10 @@ mod test {
         let fee_rate = FeeRate::from_sat_per_vb(10.0);
         let utxos: Vec<OutputGroup> = generate_same_value_utxos(100_000, 100_000)
             .into_iter()
-            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
-        let curr_available_value = utxos
-            .iter()
-            .fold(0, |acc, x| acc + x.effective_value as u64);
+        let curr_available_value = utxos.iter().fold(0, |acc, x| acc + x.effective_value);
 
         let size_of_change = 31;
         let cost_of_change = size_of_change as f32 * fee_rate.as_sat_vb();
@@ -892,7 +954,7 @@ mod test {
                 0,
                 curr_available_value,
                 20_000,
-                50.0,
+                FEE_AMOUNT,
                 cost_of_change,
             )
             .unwrap();
@@ -904,22 +966,19 @@ mod test {
         let fee_rate = FeeRate::from_sat_per_vb(1.0);
         let size_of_change = 31;
         let cost_of_change = size_of_change as f32 * fee_rate.as_sat_vb();
-        let fee_amount = 50.0;
 
         let utxos: Vec<_> = generate_same_value_utxos(50_000, 10)
             .into_iter()
-            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
         let curr_value = 0;
 
-        let curr_available_value = utxos
-            .iter()
-            .fold(0, |acc, x| acc + x.effective_value as u64);
+        let curr_available_value = utxos.iter().fold(0, |acc, x| acc + x.effective_value);
 
         // 2*(value of 1 utxo)  - 2*(1 utxo fees with 1.0sat/vbyte fee rate) -
         // cost_of_change + 5.
-        let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change.ceil() as u64 + 5;
+        let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change.ceil() as i64 + 5;
 
         let result = BranchAndBoundCoinSelection::new(size_of_change)
             .bnb(
@@ -928,12 +987,12 @@ mod test {
                 curr_value,
                 curr_available_value,
                 target_amount,
-                fee_amount,
+                FEE_AMOUNT,
                 cost_of_change,
             )
             .unwrap();
-        assert_eq!(result.fee_amount, 186.0);
-        assert_eq!(result.selected_amount, 100_000);
+        assert_eq!(result.selected_amount(), 100_000);
+        assert_eq!(result.fee_amount, 186);
     }
 
     // TODO: bnb() function should be optimized, and this test should be done with more utxos
@@ -946,17 +1005,17 @@ mod test {
         for _ in 0..200 {
             let optional_utxos: Vec<_> = generate_random_utxos(&mut rng, 40)
                 .into_iter()
-                .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+                .map(|u| OutputGroup::new(u, fee_rate))
                 .collect();
 
             let curr_value = 0;
 
             let curr_available_value = optional_utxos
                 .iter()
-                .fold(0, |acc, x| acc + x.effective_value as u64);
+                .fold(0, |acc, x| acc + x.effective_value);
 
-            let target_amount = optional_utxos[3].effective_value as u64
-                + optional_utxos[23].effective_value as u64;
+            let target_amount =
+                optional_utxos[3].effective_value + optional_utxos[23].effective_value;
 
             let result = BranchAndBoundCoinSelection::new(0)
                 .bnb(
@@ -965,11 +1024,11 @@ mod test {
                     curr_value,
                     curr_available_value,
                     target_amount,
-                    0.0,
+                    0,
                     0.0,
                 )
                 .unwrap();
-            assert_eq!(result.selected_amount, target_amount);
+            assert_eq!(result.selected_amount(), target_amount as u64);
         }
     }
 
@@ -983,21 +1042,18 @@ mod test {
         let fee_rate = FeeRate::from_sat_per_vb(1.0);
         let utxos: Vec<OutputGroup> = utxos
             .into_iter()
-            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
         let result = BranchAndBoundCoinSelection::default().single_random_draw(
             vec![],
             utxos,
             0,
-            target_amount,
-            50.0,
+            target_amount as i64,
+            FEE_AMOUNT,
         );
 
-        assert!(result.selected_amount > target_amount);
-        assert_eq!(
-            result.fee_amount,
-            50.0 + result.selected.len() as f32 * 68.0
-        );
+        assert!(result.selected_amount() > target_amount);
+        assert_eq!(result.fee_amount, (50 + result.selected.len() * 68) as u64);
     }
 }

src/wallet/export.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Wallet export
 //!
@@ -141,7 +128,7 @@ impl WalletExport {
             Ok(txs) => {
                 let mut heights = txs
                     .into_iter()
-                    .map(|tx| tx.height.unwrap_or(0))
+                    .map(|tx| tx.confirmation_time.map(|c| c.height).unwrap_or(0))
                     .collect::<Vec<_>>();
                 heights.sort_unstable();
 
@@ -225,6 +212,7 @@ mod test {
     use crate::database::{memory::MemoryDatabase, BatchOperations};
     use crate::types::TransactionDetails;
     use crate::wallet::Wallet;
+    use crate::ConfirmationTime;
 
     fn get_test_db() -> MemoryDatabase {
         let mut db = MemoryDatabase::new();
@@ -234,11 +222,15 @@ mod test {
                 "4ddff1fa33af17f377f62b72357b43107c19110a8009b36fb832af505efed98a",
             )
             .unwrap(),
-            timestamp: 12345678,
+
             received: 100_000,
             sent: 0,
-            fees: 500,
-            height: Some(5000),
+            fee: Some(500),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 12345678,
+                height: 5000,
+            }),
+            verified: true,
         })
         .unwrap();
 

src/wallet/signer.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Generalized signers
 //!
@@ -105,7 +92,7 @@ use bitcoin::blockdata::opcodes;
 use bitcoin::blockdata::script::Builder as ScriptBuilder;
 use bitcoin::hashes::{hash160, Hash};
 use bitcoin::secp256k1::{Message, Secp256k1};
-use bitcoin::util::bip32::{ExtendedPrivKey, Fingerprint};
+use bitcoin::util::bip32::{ChildNumber, DerivationPath, ExtendedPrivKey, Fingerprint};
 use bitcoin::util::{bip143, psbt};
 use bitcoin::{PrivateKey, Script, SigHash, SigHashType};
 
@@ -159,7 +146,13 @@ pub enum SignerError {
     /// The `witness_script` field of the transaction is requied to sign this input
     MissingWitnessScript,
     /// The fingerprint and derivation path are missing from the psbt input
-    MissingHDKeypath,
+    MissingHdKeypath,
+    /// The psbt contains a non-`SIGHASH_ALL` sighash in one of its input and the user hasn't
+    /// explicitly allowed them
+    ///
+    /// To enable signing transactions with non-standard sighashes set
+    /// [`SignOptions::allow_all_sighashes`] to `true`.
+    NonStandardSighash,
 }
 
 impl fmt::Display for SignerError {
@@ -219,11 +212,17 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
-        let (public_key, deriv_path) = match psbt.inputs[input_index]
+        if psbt.inputs[input_index].final_script_sig.is_some()
+            || psbt.inputs[input_index].final_script_witness.is_some()
+        {
+            return Ok(());
+        }
+
+        let (public_key, full_path) = match psbt.inputs[input_index]
             .bip32_derivation
             .iter()
             .filter_map(|(pk, &(fingerprint, ref path))| {
-                if self.matches(&(fingerprint, path.clone()), &secp).is_some() {
+                if self.matches(&(fingerprint, path.clone()), secp).is_some() {
                     Some((pk, path))
                 } else {
                     None
@@ -235,8 +234,18 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
             None => return Ok(()),
         };
 
-        let derived_key = self.xkey.derive_priv(&secp, &deriv_path).unwrap();
-        if &derived_key.private_key.public_key(&secp) != public_key {
+        let derived_key = match self.origin.clone() {
+            Some((_fingerprint, origin_path)) => {
+                let deriv_path = DerivationPath::from(
+                    &full_path.into_iter().cloned().collect::<Vec<ChildNumber>>()
+                        [origin_path.len()..],
+                );
+                self.xkey.derive_priv(secp, &deriv_path).unwrap()
+            }
+            None => self.xkey.derive_priv(secp, &full_path).unwrap(),
+        };
+
+        if &derived_key.private_key.public_key(secp) != public_key {
             Err(SignerError::InvalidKey)
         } else {
             derived_key.private_key.sign(psbt, Some(input_index), secp)
@@ -248,7 +257,7 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
     }
 
     fn id(&self, secp: &SecpCtx) -> SignerId {
-        SignerId::from(self.root_fingerprint(&secp))
+        SignerId::from(self.root_fingerprint(secp))
     }
 
     fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
@@ -264,11 +273,17 @@ impl Signer for PrivateKey {
         secp: &SecpCtx,
     ) -> Result<(), SignerError> {
         let input_index = input_index.unwrap();
-        if input_index >= psbt.inputs.len() {
+        if input_index >= psbt.inputs.len() || input_index >= psbt.global.unsigned_tx.input.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
-        let pubkey = self.public_key(&secp);
+        if psbt.inputs[input_index].final_script_sig.is_some()
+            || psbt.inputs[input_index].final_script_witness.is_some()
+        {
+            return Ok(());
+        }
+
+        let pubkey = self.public_key(secp);
         if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
             return Ok(());
         }
@@ -430,6 +445,50 @@ impl SignersContainer {
     }
 }
 
+/// Options for a software signer
+///
+/// Adjust the behavior of our software signers and the way a transaction is finalized
+#[derive(Debug, Clone)]
+pub struct SignOptions {
+    /// Whether the signer should trust the `witness_utxo`, if the `non_witness_utxo` hasn't been
+    /// provided
+    ///
+    /// Defaults to `false` to mitigate the "SegWit bug" which chould trick the wallet into
+    /// paying a fee larger than expected.
+    ///
+    /// Some wallets, especially if relatively old, might not provide the `non_witness_utxo` for
+    /// SegWit transactions in the PSBT they generate: in those cases setting this to `true`
+    /// should correctly produce a signature, at the expense of an increased trust in the creator
+    /// of the PSBT.
+    ///
+    /// For more details see: <https://blog.trezor.io/details-of-firmware-updates-for-trezor-one-version-1-9-1-and-trezor-model-t-version-2-3-1-1eba8f60f2dd>
+    pub trust_witness_utxo: bool,
+
+    /// Whether the wallet should assume a specific height has been reached when trying to finalize
+    /// a transaction
+    ///
+    /// The wallet will only "use" a timelock to satisfy the spending policy of an input if the
+    /// timelock height has already been reached. This option allows overriding the "current height" to let the
+    /// wallet use timelocks in the future to spend a coin.
+    pub assume_height: Option<u32>,
+
+    /// Whether the signer should use the `sighash_type` set in the PSBT when signing, no matter
+    /// what its value is
+    ///
+    /// Defaults to `false` which will only allow signing using `SIGHASH_ALL`.
+    pub allow_all_sighashes: bool,
+}
+
+impl Default for SignOptions {
+    fn default() -> Self {
+        SignOptions {
+            trust_witness_utxo: false,
+            assume_height: None,
+            allow_all_sighashes: false,
+        }
+    }
+}
+
 pub(crate) trait ComputeSighash {
     fn sighash(
         psbt: &psbt::PartiallySignedTransaction,
@@ -442,7 +501,7 @@ impl ComputeSighash for Legacy {
         psbt: &psbt::PartiallySignedTransaction,
         input_index: usize,
     ) -> Result<(SigHash, SigHashType), SignerError> {
-        if input_index >= psbt.inputs.len() {
+        if input_index >= psbt.inputs.len() || input_index >= psbt.global.unsigned_tx.input.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
@@ -490,32 +549,49 @@ impl ComputeSighash for Segwitv0 {
         psbt: &psbt::PartiallySignedTransaction,
         input_index: usize,
     ) -> Result<(SigHash, SigHashType), SignerError> {
-        if input_index >= psbt.inputs.len() {
+        if input_index >= psbt.inputs.len() || input_index >= psbt.global.unsigned_tx.input.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
         let psbt_input = &psbt.inputs[input_index];
+        let tx_input = &psbt.global.unsigned_tx.input[input_index];
 
         let sighash = psbt_input.sighash_type.unwrap_or(SigHashType::All);
 
-        let witness_utxo = psbt_input
-            .witness_utxo
-            .as_ref()
-            .ok_or(SignerError::MissingNonWitnessUtxo)?;
-        let value = witness_utxo.value;
+        // Always try first with the non-witness utxo
+        let utxo = if let Some(prev_tx) = &psbt_input.non_witness_utxo {
+            // Check the provided prev-tx
+            if prev_tx.txid() != tx_input.previous_output.txid {
+                return Err(SignerError::InvalidNonWitnessUtxo);
+            }
+
+            // The output should be present, if it's missing the `non_witness_utxo` is invalid
+            prev_tx
+                .output
+                .get(tx_input.previous_output.vout as usize)
+                .ok_or(SignerError::InvalidNonWitnessUtxo)?
+        } else if let Some(witness_utxo) = &psbt_input.witness_utxo {
+            // Fallback to the witness_utxo. If we aren't allowed to use it, signing should fail
+            // before we get to this point
+            witness_utxo
+        } else {
+            // Nothing has been provided
+            return Err(SignerError::MissingNonWitnessUtxo);
+        };
+        let value = utxo.value;
 
         let script = match psbt_input.witness_script {
             Some(ref witness_script) => witness_script.clone(),
             None => {
-                if witness_utxo.script_pubkey.is_v0_p2wpkh() {
-                    p2wpkh_script_code(&witness_utxo.script_pubkey)
+                if utxo.script_pubkey.is_v0_p2wpkh() {
+                    p2wpkh_script_code(&utxo.script_pubkey)
                 } else if psbt_input
                     .redeem_script
                     .as_ref()
                     .map(Script::is_v0_p2wpkh)
                     .unwrap_or(false)
                 {
-                    p2wpkh_script_code(&psbt_input.redeem_script.as_ref().unwrap())
+                    p2wpkh_script_code(psbt_input.redeem_script.as_ref().unwrap())
                 } else {
                     return Err(SignerError::MissingWitnessScript);
                 }
@@ -569,6 +645,11 @@ mod signers_container_tests {
     use miniscript::ScriptContext;
     use std::str::FromStr;
 
+    fn is_equal(this: &Arc<dyn Signer>, that: &Arc<DummySigner>) -> bool {
+        let secp = Secp256k1::new();
+        this.id(&secp) == that.id(&secp)
+    }
+
     // Signers added with the same ordering (like `Ordering::default`) created from `KeyMap`
     // should be preserved and not overwritten.
     // This happens usually when a set of signers is created from a descriptor with private keys.
@@ -593,73 +674,58 @@ mod signers_container_tests {
     #[test]
     fn signers_sorted_by_ordering() {
         let mut signers = SignersContainer::new();
-        let signer1 = Arc::new(DummySigner);
-        let signer2 = Arc::new(DummySigner);
-        let signer3 = Arc::new(DummySigner);
+        let signer1 = Arc::new(DummySigner { number: 1 });
+        let signer2 = Arc::new(DummySigner { number: 2 });
+        let signer3 = Arc::new(DummySigner { number: 3 });
 
-        signers.add_external(
-            SignerId::Fingerprint(b"cafe"[..].into()),
-            SignerOrdering(1),
-            signer1.clone(),
-        );
-        signers.add_external(
-            SignerId::Fingerprint(b"babe"[..].into()),
-            SignerOrdering(2),
-            signer2.clone(),
-        );
-        signers.add_external(
-            SignerId::Fingerprint(b"feed"[..].into()),
-            SignerOrdering(3),
-            signer3.clone(),
-        );
+        // Mixed order insertions verifies we are not inserting at head or tail.
+        signers.add_external(SignerId::Dummy(2), SignerOrdering(2), signer2.clone());
+        signers.add_external(SignerId::Dummy(1), SignerOrdering(1), signer1.clone());
+        signers.add_external(SignerId::Dummy(3), SignerOrdering(3), signer3.clone());
 
         // Check that signers are sorted from lowest to highest ordering
         let signers = signers.signers();
-        assert_eq!(Arc::as_ptr(signers[0]), Arc::as_ptr(&signer1));
-        assert_eq!(Arc::as_ptr(signers[1]), Arc::as_ptr(&signer2));
-        assert_eq!(Arc::as_ptr(signers[2]), Arc::as_ptr(&signer3));
+
+        assert!(is_equal(signers[0], &signer1));
+        assert!(is_equal(signers[1], &signer2));
+        assert!(is_equal(signers[2], &signer3));
     }
 
     #[test]
     fn find_signer_by_id() {
         let mut signers = SignersContainer::new();
-        let signer1: Arc<dyn Signer> = Arc::new(DummySigner);
-        let signer2: Arc<dyn Signer> = Arc::new(DummySigner);
-        let signer3: Arc<dyn Signer> = Arc::new(DummySigner);
-        let signer4: Arc<dyn Signer> = Arc::new(DummySigner);
+        let signer1 = Arc::new(DummySigner { number: 1 });
+        let signer2 = Arc::new(DummySigner { number: 2 });
+        let signer3 = Arc::new(DummySigner { number: 3 });
+        let signer4 = Arc::new(DummySigner { number: 3 }); // Same ID as `signer3` but will use lower ordering.
 
-        let id1 = SignerId::Fingerprint(b"cafe"[..].into());
-        let id2 = SignerId::Fingerprint(b"babe"[..].into());
-        let id3 = SignerId::Fingerprint(b"feed"[..].into());
-        let id_nonexistent = SignerId::Fingerprint(b"fefe"[..].into());
+        let id1 = SignerId::Dummy(1);
+        let id2 = SignerId::Dummy(2);
+        let id3 = SignerId::Dummy(3);
+        let id_nonexistent = SignerId::Dummy(999);
 
         signers.add_external(id1.clone(), SignerOrdering(1), signer1.clone());
         signers.add_external(id2.clone(), SignerOrdering(2), signer2.clone());
         signers.add_external(id3.clone(), SignerOrdering(3), signer3.clone());
 
-        assert!(
-            matches!(signers.find(id1), Some(signer) if Arc::as_ptr(&signer1) == Arc::as_ptr(signer))
-        );
-        assert!(
-            matches!(signers.find(id2), Some(signer) if Arc::as_ptr(&signer2) == Arc::as_ptr(signer))
-        );
-        assert!(
-            matches!(signers.find(id3.clone()), Some(signer) if Arc::as_ptr(&signer3) == Arc::as_ptr(signer))
-        );
+        assert!(matches!(signers.find(id1), Some(signer) if is_equal(signer, &signer1)));
+        assert!(matches!(signers.find(id2), Some(signer) if is_equal(signer, &signer2)));
+        assert!(matches!(signers.find(id3.clone()), Some(signer) if is_equal(signer, &signer3)));
 
         // The `signer4` has the same ID as `signer3` but lower ordering.
         // It should be found by `id3` instead of `signer3`.
         signers.add_external(id3.clone(), SignerOrdering(2), signer4.clone());
-        assert!(
-            matches!(signers.find(id3), Some(signer) if Arc::as_ptr(&signer4) == Arc::as_ptr(signer))
-        );
+        assert!(matches!(signers.find(id3), Some(signer) if is_equal(signer, &signer4)));
 
         // Can't find anything with ID that doesn't exist
         assert!(matches!(signers.find(id_nonexistent), None));
     }
 
-    #[derive(Debug)]
-    struct DummySigner;
+    #[derive(Debug, Clone, Copy)]
+    struct DummySigner {
+        number: u64,
+    }
+
     impl Signer for DummySigner {
         fn sign(
             &self,
@@ -671,7 +737,7 @@ mod signers_container_tests {
         }
 
         fn id(&self, _secp: &SecpCtx) -> SignerId {
-            SignerId::Dummy(42)
+            SignerId::Dummy(self.number)
         }
 
         fn sign_whole_tx(&self) -> bool {

src/wallet/time.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Cross-platform time
 //!

src/wallet/tx_builder.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Transaction builder
 //!
@@ -54,15 +41,15 @@ use std::collections::HashSet;
 use std::default::Default;
 use std::marker::PhantomData;
 
-use bitcoin::util::psbt::PartiallySignedTransaction as PSBT;
+use bitcoin::util::psbt::{self, PartiallySignedTransaction as Psbt};
 use bitcoin::{OutPoint, Script, SigHashType, Transaction};
 
 use miniscript::descriptor::DescriptorTrait;
 
 use super::coin_selection::{CoinSelectionAlgorithm, DefaultCoinSelectionAlgorithm};
-use crate::{database::BatchDatabase, Error, Wallet};
+use crate::{database::BatchDatabase, Error, Utxo, Wallet};
 use crate::{
-    types::{FeeRate, KeychainKind, UTXO},
+    types::{FeeRate, KeychainKind, LocalUtxo, WeightedUtxo},
     TransactionDetails,
 };
 /// Context in which the [`TxBuilder`] is valid
@@ -100,9 +87,9 @@ impl TxBuilderContext for BumpFee {}
 /// let (psbt1, details) = {
 ///     let mut builder = wallet.build_tx();
 ///     builder
-///        .ordering(TxOrdering::Untouched)
-///        .add_recipient(addr1.script_pubkey(), 50_000)
-///        .add_recipient(addr2.script_pubkey(), 50_000);
+///         .ordering(TxOrdering::Untouched)
+///         .add_recipient(addr1.script_pubkey(), 50_000)
+///         .add_recipient(addr2.script_pubkey(), 50_000);
 ///     builder.finish()?
 /// };
 ///
@@ -116,7 +103,10 @@ impl TxBuilderContext for BumpFee {}
 ///     builder.finish()?
 /// };
 ///
-/// assert_eq!(psbt1.global.unsigned_tx.output[..2], psbt2.global.unsigned_tx.output[..2]);
+/// assert_eq!(
+///     psbt1.global.unsigned_tx.output[..2],
+///     psbt2.global.unsigned_tx.output[..2]
+/// );
 /// # Ok::<(), bdk::Error>(())
 /// ```
 ///
@@ -129,12 +119,9 @@ impl TxBuilderContext for BumpFee {}
 /// [`build_fee_bump`]: Wallet::build_fee_bump
 /// [`finish`]: Self::finish
 /// [`coin_selection`]: Self::coin_selection
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct TxBuilder<'a, B, D, Cs, Ctx> {
     pub(crate) wallet: &'a Wallet<B, D>,
-    // params and coin_selection are Options not becasue they are optionally set (they are always
-    // there) but because `.finish()` uses `Option::take` to get an owned value from a &mut self.
-    // They are only `None` after `.finish()` is called.
     pub(crate) params: TxParams,
     pub(crate) coin_selection: Cs,
     pub(crate) phantom: PhantomData<Ctx>,
@@ -146,20 +133,20 @@ pub struct TxBuilder<'a, B, D, Cs, Ctx> {
 pub(crate) struct TxParams {
     pub(crate) recipients: Vec<(Script, u64)>,
     pub(crate) drain_wallet: bool,
-    pub(crate) single_recipient: Option<Script>,
+    pub(crate) drain_to: Option<Script>,
     pub(crate) fee_policy: Option<FeePolicy>,
     pub(crate) internal_policy_path: Option<BTreeMap<String, Vec<usize>>>,
     pub(crate) external_policy_path: Option<BTreeMap<String, Vec<usize>>>,
-    pub(crate) utxos: Vec<(UTXO, usize)>,
+    pub(crate) utxos: Vec<WeightedUtxo>,
     pub(crate) unspendable: HashSet<OutPoint>,
     pub(crate) manually_selected_only: bool,
     pub(crate) sighash: Option<SigHashType>,
     pub(crate) ordering: TxOrdering,
     pub(crate) locktime: Option<u32>,
-    pub(crate) rbf: Option<RBFValue>,
+    pub(crate) rbf: Option<RbfValue>,
     pub(crate) version: Option<Version>,
     pub(crate) change_policy: ChangeSpendPolicy,
-    pub(crate) force_non_witness_utxo: bool,
+    pub(crate) only_witness_utxo: bool,
     pub(crate) add_global_xpubs: bool,
     pub(crate) include_output_redeem_witness_script: bool,
     pub(crate) bumping_fee: Option<PreviousFee>,
@@ -183,6 +170,17 @@ impl std::default::Default for FeePolicy {
     }
 }
 
+impl<'a, Cs: Clone, Ctx, B, D> Clone for TxBuilder<'a, B, D, Cs, Ctx> {
+    fn clone(&self) -> Self {
+        TxBuilder {
+            wallet: self.wallet,
+            params: self.params.clone(),
+            coin_selection: self.coin_selection.clone(),
+            phantom: PhantomData,
+        }
+    }
+}
+
 // methods supported by both contexts, for any CoinSelectionAlgorithm
 impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext>
     TxBuilder<'a, B, D, Cs, Ctx>
@@ -251,7 +249,8 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderConte
     /// let mut path = BTreeMap::new();
     /// path.insert("aabbccdd".to_string(), vec![0, 1]);
     ///
-    /// let builder = wallet.build_tx()
+    /// let builder = wallet
+    ///     .build_tx()
     ///     .add_recipient(to_address.script_pubkey(), 50_000)
     ///     .policy_path(path, KeychainKind::External);
     ///
@@ -280,13 +279,16 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderConte
     pub fn add_utxos(&mut self, outpoints: &[OutPoint]) -> Result<&mut Self, Error> {
         let utxos = outpoints
             .iter()
-            .map(|outpoint| self.wallet.get_utxo(*outpoint)?.ok_or(Error::UnknownUTXO))
+            .map(|outpoint| self.wallet.get_utxo(*outpoint)?.ok_or(Error::UnknownUtxo))
             .collect::<Result<Vec<_>, _>>()?;
 
         for utxo in utxos {
             let descriptor = self.wallet.get_descriptor_for_keychain(utxo.keychain);
             let satisfaction_weight = descriptor.max_satisfaction_weight().unwrap();
-            self.params.utxos.push((utxo, satisfaction_weight));
+            self.params.utxos.push(WeightedUtxo {
+                satisfaction_weight,
+                utxo: Utxo::Local(utxo),
+            });
         }
 
         Ok(self)
@@ -300,6 +302,84 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderConte
         self.add_utxos(&[outpoint])
     }
 
+    /// Add a foreign UTXO i.e. a UTXO not owned by this wallet.
+    ///
+    /// At a minimum to add a foreign UTXO we need:
+    ///
+    /// 1. `outpoint`: To add it to the raw transaction.
+    /// 2. `psbt_input`: To know the value.
+    /// 3. `satisfaction_weight`: To know how much weight/vbytes the input will add to the transaction for fee calculation.
+    ///
+    /// There are several security concerns about adding foregin UTXOs that application
+    /// developers should consider. First, how do you know the value of the input is correct? If a
+    /// `non_witness_utxo` is provided in the `psbt_input` then this method implicitly verifies the
+    /// value by checking it against the transaction. If only a `witness_utxo` is provided then this
+    /// method doesn't verify the value but just takes it as a given -- it is up to you to check
+    /// that whoever sent you the `input_psbt` was not lying!
+    ///
+    /// Secondly, you must somehow provide `satisfaction_weight` of the input. Depending on your
+    /// application it may be important that this be known precisely. If not, a malicious
+    /// counterparty may fool you into putting in a value that is too low, giving the transaction a
+    /// lower than expected feerate. They could also fool you into putting a value that is too high
+    /// causing you to pay a fee that is too high. The party who is broadcasting the transaction can
+    /// of course check the real input weight matches the expected weight prior to broadcasting.
+    ///
+    /// To guarantee the `satisfaction_weight` is correct, you can require the party providing the
+    /// `psbt_input` provide a miniscript descriptor for the input so you can check it against the
+    /// `script_pubkey` and then ask it for the [`max_satisfaction_weight`].
+    ///
+    /// This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+    ///
+    /// # Errors
+    ///
+    /// This method returns errors in the following circumstances:
+    ///
+    /// 1. The `psbt_input` does not contain a `witness_utxo` or `non_witness_utxo`.
+    /// 2. The data in `non_witness_utxo` does not match what is in `outpoint`.
+    ///
+    /// Note unless you set [`only_witness_utxo`] any `psbt_input` you pass to this method must
+    /// have `non_witness_utxo` set otherwise you will get an error when [`finish`] is called.
+    ///
+    /// [`only_witness_utxo`]: Self::only_witness_utxo
+    /// [`finish`]: Self::finish
+    /// [`max_satisfaction_weight`]: miniscript::Descriptor::max_satisfaction_weight
+    pub fn add_foreign_utxo(
+        &mut self,
+        outpoint: OutPoint,
+        psbt_input: psbt::Input,
+        satisfaction_weight: usize,
+    ) -> Result<&mut Self, Error> {
+        if psbt_input.witness_utxo.is_none() {
+            match psbt_input.non_witness_utxo.as_ref() {
+                Some(tx) => {
+                    if tx.txid() != outpoint.txid {
+                        return Err(Error::Generic(
+                            "Foreign utxo outpoint does not match PSBT input".into(),
+                        ));
+                    }
+                    if tx.output.len() <= outpoint.vout as usize {
+                        return Err(Error::InvalidOutpoint(outpoint));
+                    }
+                }
+                None => {
+                    return Err(Error::Generic(
+                        "Foreign utxo missing witness_utxo or non_witness_utxo".into(),
+                    ))
+                }
+            }
+        }
+
+        self.params.utxos.push(WeightedUtxo {
+            satisfaction_weight,
+            utxo: Utxo::Foreign {
+                outpoint,
+                psbt_input: Box::new(psbt_input),
+            },
+        });
+
+        Ok(self)
+    }
+
     /// Only spend utxos added by [`add_utxo`].
     ///
     /// The wallet will **not** add additional utxos to the transaction even if they are needed to
@@ -385,12 +465,13 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderConte
         self
     }
 
-    /// Fill-in the [`psbt::Input::non_witness_utxo`](bitcoin::util::psbt::Input::non_witness_utxo) field even if the wallet only has SegWit
-    /// descriptors.
+    /// Only Fill-in the [`psbt::Input::witness_utxo`](bitcoin::util::psbt::Input::witness_utxo) field when spending from
+    /// SegWit descriptors.
     ///
-    /// This is useful for signers which always require it, like Trezor hardware wallets.
-    pub fn force_non_witness_utxo(&mut self) -> &mut Self {
-        self.params.force_non_witness_utxo = true;
+    /// This reduces the size of the PSBT, but some signers might reject them due to the lack of
+    /// the `non_witness_utxo`.
+    pub fn only_witness_utxo(&mut self) -> &mut Self {
+        self.params.only_witness_utxo = true;
         self
     }
 
@@ -441,9 +522,29 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderConte
     /// Returns the [`BIP174`] "PSBT" and summary details about the transaction.
     ///
     /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
-    pub fn finish(self) -> Result<(PSBT, TransactionDetails), Error> {
+    pub fn finish(self) -> Result<(Psbt, TransactionDetails), Error> {
         self.wallet.create_tx(self.coin_selection, self.params)
     }
+
+    /// Enable signaling RBF
+    ///
+    /// This will use the default nSequence value of `0xFFFFFFFD`.
+    pub fn enable_rbf(&mut self) -> &mut Self {
+        self.params.rbf = Some(RbfValue::Default);
+        self
+    }
+
+    /// Enable signaling RBF with a specific nSequence value
+    ///
+    /// This can cause conflicts if the wallet's descriptors contain an "older" (OP_CSV) operator
+    /// and the given `nsequence` is lower than the CSV value.
+    ///
+    /// If the `nsequence` is higher than `0xFFFFFFFD` an error will be thrown, since it would not
+    /// be a valid nSequence to signal RBF.
+    pub fn enable_rbf_with_sequence(&mut self, nsequence: u32) -> &mut Self {
+        self.params.rbf = Some(RbfValue::Value(nsequence));
+        self
+    }
 }
 
 impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>> TxBuilder<'a, B, D, Cs, CreateTx> {
@@ -459,69 +560,81 @@ impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>> TxBuilder<'a, B, D,
         self
     }
 
-    /// Set a single recipient that will get all the selected funds minus the fee. No change will
-    /// be created
+    /// Sets the address to *drain* excess coins to.
     ///
-    /// This method overrides any recipient set with [`set_recipients`](Self::set_recipients) or
-    /// [`add_recipient`](Self::add_recipient).
+    /// Usually, when there are excess coins they are sent to a change address generated by the
+    /// wallet. This option replaces the usual change address with an arbitrary `script_pubkey` of
+    /// your choosing. Just as with a change output, if the drain output is not needed (the excess
+    /// coins are too small) it will not be included in the resulting transaction. The only
+    /// difference is that it is valid to use `drain_to` without setting any ordinary recipients
+    /// with [`add_recipient`] (but it is perfectly fine to add recipients as well).
     ///
-    /// It can only be used in conjunction with [`drain_wallet`](Self::drain_wallet) to send the
-    /// entire content of the wallet (minus filters) to a single recipient or with a
-    /// list of manually selected UTXOs by enabling [`manually_selected_only`](Self::manually_selected_only)
-    /// and selecting them with or [`add_utxo`](Self::add_utxo).
+    /// When bumping the fees of a transaction made with this option, you probably want to
+    /// use [`allow_shrinking`] to allow this output to be reduced to pay for the extra fees.
     ///
-    /// When bumping the fees of a transaction made with this option, the user should remeber to
-    /// add [`maintain_single_recipient`](Self::maintain_single_recipient) to correctly update the
-    /// single output instead of adding one more for the change.
-    pub fn set_single_recipient(&mut self, recipient: Script) -> &mut Self {
-        self.params.single_recipient = Some(recipient);
-        self.params.recipients.clear();
-
-        self
-    }
-
-    /// Enable signaling RBF
+    /// # Example
     ///
-    /// This will use the default nSequence value of `0xFFFFFFFD`.
-    pub fn enable_rbf(&mut self) -> &mut Self {
-        self.params.rbf = Some(RBFValue::Default);
-        self
-    }
-
-    /// Enable signaling RBF with a specific nSequence value
+    /// `drain_to` is very useful for draining all the coins in a wallet with [`drain_wallet`] to a
+    /// single address.
     ///
-    /// This can cause conflicts if the wallet's descriptors contain an "older" (OP_CSV) operator
-    /// and the given `nsequence` is lower than the CSV value.
+    /// ```
+    /// # use std::str::FromStr;
+    /// # use bitcoin::*;
+    /// # use bdk::*;
+    /// # use bdk::wallet::tx_builder::CreateTx;
+    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// # let wallet = doctest_wallet!();
+    /// let mut tx_builder = wallet.build_tx();
     ///
-    /// If the `nsequence` is higher than `0xFFFFFFFD` an error will be thrown, since it would not
-    /// be a valid nSequence to signal RBF.
-    pub fn enable_rbf_with_sequence(&mut self, nsequence: u32) -> &mut Self {
-        self.params.rbf = Some(RBFValue::Value(nsequence));
+    /// tx_builder
+    ///     // Spend all outputs in this wallet.
+    ///     .drain_wallet()
+    ///     // Send the excess (which is all the coins minus the fee) to this address.
+    ///     .drain_to(to_address.script_pubkey())
+    ///     .fee_rate(FeeRate::from_sat_per_vb(5.0))
+    ///     .enable_rbf();
+    /// let (psbt, tx_details) = tx_builder.finish()?;
+    /// # Ok::<(), bdk::Error>(())
+    /// ```
+    ///
+    /// [`allow_shrinking`]: Self::allow_shrinking
+    /// [`add_recipient`]: Self::add_recipient
+    /// [`drain_wallet`]: Self::drain_wallet
+    pub fn drain_to(&mut self, script_pubkey: Script) -> &mut Self {
+        self.params.drain_to = Some(script_pubkey);
         self
     }
 }
 
 // methods supported only by bump_fee
 impl<'a, B, D: BatchDatabase> TxBuilder<'a, B, D, DefaultCoinSelectionAlgorithm, BumpFee> {
-    /// Bump the fees of a transaction made with [`set_single_recipient`](Self::set_single_recipient)
+    /// Explicitly tells the wallet that it is allowed to reduce the fee of the output matching this
+    /// `script_pubkey` in order to bump the transaction fee. Without specifying this the wallet
+    /// will attempt to find a change output to shrink instead.
     ///
-    /// Unless extra inputs are specified with [`add_utxo`], this flag will make
-    /// `bump_fee` reduce the value of the existing output, or fail if it would be consumed
-    /// entirely given the higher new fee rate.
+    /// **Note** that the output may shrink to below the dust limit and therefore be removed. If it is
+    /// preserved then it is currently not guaranteed to be in the same position as it was
+    /// originally.
     ///
-    /// If extra inputs are added and they are not entirely consumed in fees, a change output will not
-    /// be added; the existing output will simply grow in value.
-    ///
-    /// Fails if the transaction has more than one outputs.
-    ///
-    /// [`add_utxo`]: Self::add_utxo
-    pub fn maintain_single_recipient(&mut self) -> Result<&mut Self, Error> {
-        let mut recipients = self.params.recipients.drain(..).collect::<Vec<_>>();
-        if recipients.len() != 1 {
-            return Err(Error::SingleRecipientMultipleOutputs);
+    /// Returns an `Err` if `script_pubkey` can't be found among the recipients of the
+    /// transaction we are bumping.
+    pub fn allow_shrinking(&mut self, script_pubkey: Script) -> Result<&mut Self, Error> {
+        match self
+            .params
+            .recipients
+            .iter()
+            .position(|(recipient_script, _)| *recipient_script == script_pubkey)
+        {
+            Some(position) => {
+                self.params.recipients.remove(position);
+                self.params.drain_to = Some(script_pubkey);
+                Ok(self)
+            }
+            None => Err(Error::Generic(format!(
+                "{} was not in the original transaction",
+                script_pubkey
+            ))),
         }
-        self.params.single_recipient = Some(recipients.pop().unwrap().0);
-        Ok(self)
     }
 }
 
@@ -533,7 +646,7 @@ pub enum TxOrdering {
     /// Unchanged
     Untouched,
     /// BIP69 / Lexicographic
-    BIP69Lexicographic,
+    Bip69Lexicographic,
 }
 
 impl Default for TxOrdering {
@@ -559,7 +672,7 @@ impl TxOrdering {
 
                 tx.output.shuffle(&mut rng);
             }
-            TxOrdering::BIP69Lexicographic => {
+            TxOrdering::Bip69Lexicographic => {
                 tx.input.sort_unstable_by_key(|txin| {
                     (txin.previous_output.txid, txin.previous_output.vout)
                 });
@@ -586,16 +699,16 @@ impl Default for Version {
 ///
 /// Has a default value of `0xFFFFFFFD`
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
-pub(crate) enum RBFValue {
+pub(crate) enum RbfValue {
     Default,
     Value(u32),
 }
 
-impl RBFValue {
+impl RbfValue {
     pub(crate) fn get_value(&self) -> u32 {
         match self {
-            RBFValue::Default => 0xFFFFFFFD,
-            RBFValue::Value(v) => *v,
+            RbfValue::Default => 0xFFFFFFFD,
+            RbfValue::Value(v) => *v,
         }
     }
 }
@@ -618,7 +731,7 @@ impl Default for ChangeSpendPolicy {
 }
 
 impl ChangeSpendPolicy {
-    pub(crate) fn is_satisfied_by(&self, utxo: &UTXO) -> bool {
+    pub(crate) fn is_satisfied_by(&self, utxo: &LocalUtxo) -> bool {
         match self {
             ChangeSpendPolicy::ChangeAllowed => true,
             ChangeSpendPolicy::OnlyChange => utxo.keychain == KeychainKind::Internal,
@@ -629,12 +742,12 @@ impl ChangeSpendPolicy {
 
 #[cfg(test)]
 mod test {
-    const ORDERING_TEST_TX: &'static str = "0200000003c26f3eb7932f7acddc5ddd26602b77e7516079b03090a16e2c2f54\
-                                            85d1fd600f0100000000ffffffffc26f3eb7932f7acddc5ddd26602b77e75160\
-                                            79b03090a16e2c2f5485d1fd600f0000000000ffffffff571fb3e02278217852\
-                                            dd5d299947e2b7354a639adc32ec1fa7b82cfb5dec530e0500000000ffffffff\
-                                            03e80300000000000002aaeee80300000000000001aa200300000000000001ff\
-                                            00000000";
+    const ORDERING_TEST_TX: &str = "0200000003c26f3eb7932f7acddc5ddd26602b77e7516079b03090a16e2c2f54\
+                                    85d1fd600f0100000000ffffffffc26f3eb7932f7acddc5ddd26602b77e75160\
+                                    79b03090a16e2c2f5485d1fd600f0000000000ffffffff571fb3e02278217852\
+                                    dd5d299947e2b7354a639adc32ec1fa7b82cfb5dec530e0500000000ffffffff\
+                                    03e80300000000000002aaeee80300000000000001aa200300000000000001ff\
+                                    00000000";
     macro_rules! ordering_test_tx {
         () => {
             deserialize::<bitcoin::Transaction>(&Vec::<u8>::from_hex(ORDERING_TEST_TX).unwrap())
@@ -678,9 +791,9 @@ mod test {
         use std::str::FromStr;
 
         let original_tx = ordering_test_tx!();
-        let mut tx = original_tx.clone();
+        let mut tx = original_tx;
 
-        TxOrdering::BIP69Lexicographic.sort_tx(&mut tx);
+        TxOrdering::Bip69Lexicographic.sort_tx(&mut tx);
 
         assert_eq!(
             tx.input[0].previous_output,
@@ -709,9 +822,9 @@ mod test {
         assert_eq!(tx.output[2].script_pubkey, From::from(vec![0xAA, 0xEE]));
     }
 
-    fn get_test_utxos() -> Vec<UTXO> {
+    fn get_test_utxos() -> Vec<LocalUtxo> {
         vec![
-            UTXO {
+            LocalUtxo {
                 outpoint: OutPoint {
                     txid: Default::default(),
                     vout: 0,
@@ -719,7 +832,7 @@ mod test {
                 txout: Default::default(),
                 keychain: KeychainKind::External,
             },
-            UTXO {
+            LocalUtxo {
                 outpoint: OutPoint {
                     txid: Default::default(),
                     vout: 1,
@@ -736,9 +849,9 @@ mod test {
         let filtered = get_test_utxos()
             .into_iter()
             .filter(|u| change_spend_policy.is_satisfied_by(u))
-            .collect::<Vec<_>>();
+            .count();
 
-        assert_eq!(filtered.len(), 2);
+        assert_eq!(filtered, 2);
     }
 
     #[test]

src/wallet/utils.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use bitcoin::secp256k1::{All, Secp256k1};
 
@@ -156,8 +143,8 @@ pub struct ChunksIterator<I: Iterator> {
     size: usize,
 }
 
+#[cfg(any(feature = "electrum", feature = "esplora"))]
 impl<I: Iterator> ChunksIterator<I> {
-    #[allow(dead_code)]
     pub fn new(iter: I, size: usize) -> Self {
         ChunksIterator { iter, size }
     }
@@ -214,31 +201,31 @@ mod test {
     #[test]
     fn test_check_nsequence_rbf_msb_set() {
         let result = check_nsequence_rbf(0x80000000, 5000);
-        assert_eq!(result, false);
+        assert!(!result);
     }
 
     #[test]
     fn test_check_nsequence_rbf_lt_csv() {
         let result = check_nsequence_rbf(4000, 5000);
-        assert_eq!(result, false);
+        assert!(!result);
     }
 
     #[test]
     fn test_check_nsequence_rbf_different_unit() {
         let result = check_nsequence_rbf(SEQUENCE_LOCKTIME_TYPE_FLAG + 5000, 5000);
-        assert_eq!(result, false);
+        assert!(!result);
     }
 
     #[test]
     fn test_check_nsequence_rbf_mask() {
         let result = check_nsequence_rbf(0x3f + 10_000, 5000);
-        assert_eq!(result, true);
+        assert!(result);
     }
 
     #[test]
     fn test_check_nsequence_rbf_same_unit_blocks() {
         let result = check_nsequence_rbf(10_000, 5000);
-        assert_eq!(result, true);
+        assert!(result);
     }
 
     #[test]
@@ -247,25 +234,25 @@ mod test {
             SEQUENCE_LOCKTIME_TYPE_FLAG + 10_000,
             SEQUENCE_LOCKTIME_TYPE_FLAG + 5000,
         );
-        assert_eq!(result, true);
+        assert!(result);
     }
 
     #[test]
     fn test_check_nlocktime_lt_cltv() {
         let result = check_nlocktime(4000, 5000);
-        assert_eq!(result, false);
+        assert!(!result);
     }
 
     #[test]
     fn test_check_nlocktime_different_unit() {
         let result = check_nlocktime(BLOCKS_TIMELOCK_THRESHOLD + 5000, 5000);
-        assert_eq!(result, false);
+        assert!(!result);
     }
 
     #[test]
     fn test_check_nlocktime_same_unit_blocks() {
         let result = check_nlocktime(10_000, 5000);
-        assert_eq!(result, true);
+        assert!(result);
     }
 
     #[test]
@@ -274,6 +261,6 @@ mod test {
             BLOCKS_TIMELOCK_THRESHOLD + 10_000,
             BLOCKS_TIMELOCK_THRESHOLD + 5000,
         );
-        assert_eq!(result, true);
+        assert!(result);
     }
 }

src/wallet/verify.rs

@@ -0,0 +1,185 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Verify transactions against the consensus rules
+
+use std::collections::HashMap;
+use std::fmt;
+
+use bitcoin::consensus::serialize;
+use bitcoin::{OutPoint, Transaction, Txid};
+
+use crate::blockchain::Blockchain;
+use crate::database::Database;
+use crate::error::Error;
+
+/// Verify a transaction against the consensus rules
+///
+/// This function uses [`bitcoinconsensus`] to verify transactions by fetching the required data
+/// either from the [`Database`] or using the [`Blockchain`].
+///
+/// Depending on the [capabilities](crate::blockchain::Blockchain::get_capabilities) of the
+/// [`Blockchain`] backend, the method could fail when called with old "historical" transactions or
+/// with unconfirmed transactions that have been evicted from the backend's memory.
+pub fn verify_tx<D: Database, B: Blockchain>(
+    tx: &Transaction,
+    database: &D,
+    blockchain: &B,
+) -> Result<(), VerifyError> {
+    log::debug!("Verifying {}", tx.txid());
+
+    let serialized_tx = serialize(tx);
+    let mut tx_cache = HashMap::<_, Transaction>::new();
+
+    for (index, input) in tx.input.iter().enumerate() {
+        let prev_tx = if let Some(prev_tx) = tx_cache.get(&input.previous_output.txid) {
+            prev_tx.clone()
+        } else if let Some(prev_tx) = database.get_raw_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else if let Some(prev_tx) = blockchain.get_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else {
+            return Err(VerifyError::MissingInputTx(input.previous_output.txid));
+        };
+
+        let spent_output = prev_tx
+            .output
+            .get(input.previous_output.vout as usize)
+            .ok_or(VerifyError::InvalidInput(input.previous_output))?;
+
+        bitcoinconsensus::verify(
+            &spent_output.script_pubkey.to_bytes(),
+            spent_output.value,
+            &serialized_tx,
+            index,
+        )?;
+
+        // Since we have a local cache we might as well cache stuff from the db, as it will very
+        // likely decrease latency compared to reading from disk or performing an SQL query.
+        tx_cache.insert(prev_tx.txid(), prev_tx);
+    }
+
+    Ok(())
+}
+
+/// Error during validation of a tx agains the consensus rules
+#[derive(Debug)]
+pub enum VerifyError {
+    /// The transaction being spent is not available in the database or the blockchain client
+    MissingInputTx(Txid),
+    /// The transaction being spent doesn't have the requested output
+    InvalidInput(OutPoint),
+
+    /// Consensus error
+    Consensus(bitcoinconsensus::Error),
+
+    /// Generic error
+    ///
+    /// It has to be wrapped in a `Box` since `Error` has a variant that contains this enum
+    Global(Box<Error>),
+}
+
+impl fmt::Display for VerifyError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for VerifyError {}
+
+impl From<Error> for VerifyError {
+    fn from(other: Error) -> Self {
+        VerifyError::Global(Box::new(other))
+    }
+}
+impl_error!(bitcoinconsensus::Error, Consensus, VerifyError);
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashSet;
+
+    use bitcoin::consensus::encode::deserialize;
+    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::{Transaction, Txid};
+
+    use crate::blockchain::{Blockchain, Capability, Progress};
+    use crate::database::{BatchDatabase, BatchOperations, MemoryDatabase};
+    use crate::FeeRate;
+
+    use super::*;
+
+    struct DummyBlockchain;
+
+    impl Blockchain for DummyBlockchain {
+        fn get_capabilities(&self) -> HashSet<Capability> {
+            Default::default()
+        }
+        fn setup<D: BatchDatabase, P: 'static + Progress>(
+            &self,
+            _database: &mut D,
+            _progress_update: P,
+        ) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_tx(&self, _txid: &Txid) -> Result<Option<Transaction>, Error> {
+            Ok(None)
+        }
+        fn broadcast(&self, _tx: &Transaction) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_height(&self) -> Result<u32, Error> {
+            Ok(42)
+        }
+        fn estimate_fee(&self, _target: usize) -> Result<FeeRate, Error> {
+            Ok(FeeRate::default_min_relay_fee())
+        }
+    }
+
+    #[test]
+    fn test_verify_fail_unsigned_tx() {
+        // https://blockstream.info/tx/95da344585fcf2e5f7d6cbf2c3df2dcce84f9196f7a7bb901a43275cd6eb7c3f
+        let prev_tx: Transaction = deserialize(&Vec::<u8>::from_hex("020000000101192dea5e66d444380e106f8e53acb171703f00d43fb6b3ae88ca5644bdb7e1000000006b48304502210098328d026ce138411f957966c1cf7f7597ccbb170f5d5655ee3e9f47b18f6999022017c3526fc9147830e1340e04934476a3d1521af5b4de4e98baf49ec4c072079e01210276f847f77ec8dd66d78affd3c318a0ed26d89dab33fa143333c207402fcec352feffffff023d0ac203000000001976a9144bfbaf6afb76cc5771bc6404810d1cc041a6933988aca4b956050000000017a91494d5543c74a3ee98e0cf8e8caef5dc813a0f34b48768cb0700").unwrap()).unwrap();
+        // https://blockstream.info/tx/aca326a724eda9a461c10a876534ecd5ae7b27f10f26c3862fb996f80ea2d45d
+        let signed_tx: Transaction = deserialize(&Vec::<u8>::from_hex("02000000013f7cebd65c27431a90bba7f796914fe8cc2ddfc3f2cbd6f7e5f2fc854534da95000000006b483045022100de1ac3bcdfb0332207c4a91f3832bd2c2915840165f876ab47c5f8996b971c3602201c6c053d750fadde599e6f5c4e1963df0f01fc0d97815e8157e3d59fe09ca30d012103699b464d1d8bc9e47d4fb1cdaa89a1c5783d68363c4dbc4b524ed3d857148617feffffff02836d3c01000000001976a914fc25d6d5c94003bf5b0c7b640a248e2c637fcfb088ac7ada8202000000001976a914fbed3d9b11183209a57999d54d59f67c019e756c88ac6acb0700").unwrap()).unwrap();
+
+        let mut database = MemoryDatabase::new();
+        let blockchain = DummyBlockchain;
+
+        let mut unsigned_tx = signed_tx.clone();
+        for input in &mut unsigned_tx.input {
+            input.script_sig = Default::default();
+            input.witness = Default::default();
+        }
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail with missing input tx");
+        assert!(
+            matches!(result, Err(VerifyError::MissingInputTx(txid)) if txid == prev_tx.txid()),
+            "Error should be a `MissingInputTx` error"
+        );
+
+        // insert the prev_tx
+        database.set_raw_tx(&prev_tx).unwrap();
+
+        let result = verify_tx(&unsigned_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail since the TX is unsigned");
+        assert!(
+            matches!(result, Err(VerifyError::Consensus(_))),
+            "Error should be a `Consensus` error"
+        );
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(
+            result.is_ok(),
+            "Should work since the TX is correctly signed"
+        );
+    }
+}

testutils-macros/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "bdk-testutils-macros"
-version = "0.3.0"
-authors = ["Alekos Filini <alekos.filini@gmail.com>"]
-edition = "2018"
-homepage = "https://bitcoindevkit.org"
-repository = "https://github.com/bitcoindevkit/bdk"
-documentation = "https://docs.rs/bdk-testutils-macros"
-description = "Supporting testing macros for `bdk`"
-keywords = ["bdk"]
-license = "MIT"
-
-[lib]
-proc-macro = true
-name = "testutils_macros"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-syn = { version = "1.0", features = ["parsing", "full"] }
-proc-macro2 = "1.0"
-quote = "1.0"
-
-[features]
-debug = ["syn/extra-traits"]

testutils-macros/src/lib.rs

@@ -1,565 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-#[macro_use]
-extern crate quote;
-
-use proc_macro::TokenStream;
-
-use syn::spanned::Spanned;
-use syn::{parse, parse2, Ident, ReturnType};
-
-#[proc_macro_attribute]
-pub fn bdk_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenStream {
-    let root_ident = if !attr.is_empty() {
-        match parse::<syn::ExprPath>(attr) {
-            Ok(parsed) => parsed,
-            Err(e) => {
-                let error_string = e.to_string();
-                return (quote! {
-                    compile_error!("Invalid crate path: {:?}", #error_string)
-                })
-                .into();
-            }
-        }
-    } else {
-        parse2::<syn::ExprPath>(quote! { bdk }).unwrap()
-    };
-
-    match parse::<syn::ItemFn>(item) {
-        Err(_) => (quote! {
-            compile_error!("#[bdk_blockchain_tests] can only be used on `fn`s")
-        })
-        .into(),
-        Ok(parsed) => {
-            let parsed_sig_ident = parsed.sig.ident.clone();
-            let mod_name = Ident::new(
-                &format!("generated_tests_{}", parsed_sig_ident.to_string()),
-                parsed.span(),
-            );
-
-            let return_type = match parsed.sig.output {
-                ReturnType::Type(_, ref t) => t.clone(),
-                ReturnType::Default => {
-                    return (quote! {
-                        compile_error!("The tagged function must return a type that impl `Blockchain`")
-                    }).into();
-                }
-            };
-
-            let output = quote! {
-
-            #parsed
-
-            mod #mod_name {
-                use bitcoin::Network;
-
-                use miniscript::Descriptor;
-
-                use testutils::{TestClient, serial};
-
-                use #root_ident::blockchain::{Blockchain, noop_progress};
-                use #root_ident::descriptor::ExtendedDescriptor;
-                use #root_ident::database::MemoryDatabase;
-                use #root_ident::types::KeychainKind;
-                use #root_ident::{Wallet, TxBuilder, FeeRate};
-
-                use super::*;
-
-                fn get_blockchain() -> #return_type {
-                    #parsed_sig_ident()
-                }
-
-                fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<#return_type, MemoryDatabase> {
-                    Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
-                }
-
-                fn init_single_sig() -> (Wallet<#return_type, MemoryDatabase>, (String, Option<String>), TestClient) {
-                    let descriptors = testutils! {
-                        @descriptors ( "wpkh(Alice)" ) ( "wpkh(Alice)" ) ( @keys ( "Alice" => (@generate_xprv "/44'/0'/0'/0/*", "/44'/0'/0'/1/*") ) )
-                    };
-
-                    let test_client = TestClient::new();
-                    let wallet = get_wallet_from_descriptors(&descriptors);
-
-                    (wallet, descriptors, test_client)
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_simple() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let tx = testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    };
-                    println!("{:?}", tx);
-                    let txid = test_client.receive(tx);
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.received, 50_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_stop_gap_20() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 5) => 50_000 )
-                    });
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 25) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 100_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_before_and_after_receive() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 0);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_multiple_outputs_same_tx() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000, (@external descriptors, 5) => 30_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 105_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 3);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.received, 105_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_receive_multi() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 5) => 25_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 2);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_address_reuse() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 25_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_receive_rbf_replaced() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) ( @replaceable true )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.received, 50_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-
-                    let new_txid = test_client.bump_fee(&txid);
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, new_txid);
-                    assert_eq!(list_tx_item.received, 50_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_reorg_block() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) ( @confirmations 1 ) ( @replaceable true )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert!(list_tx_item.height.is_some());
-
-                    // Invalidate 1 block
-                    test_client.invalidate(1);
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_after_send() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    println!("{}", descriptors.0);
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let mut builder = wallet.build_tx();
-                    builder.add_recipient(node_addr.script_pubkey(), 25_000);
-                    let (psbt, details) = builder.finish().unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    let tx = psbt.extract_tx();
-                    println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
-                    wallet.broadcast(tx).unwrap();
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_outgoing_from_scratch() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    let received_txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let mut builder = wallet.build_tx();
-                    builder.add_recipient(node_addr.script_pubkey(), 25_000);
-                    let (psbt, details) = builder.finish().unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    // empty wallet
-                    let wallet = get_wallet_from_descriptors(&descriptors);
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
-
-                    let received = tx_map.get(&received_txid).unwrap();
-                    assert_eq!(received.received, 50_000);
-                    assert_eq!(received.sent, 0);
-
-                    let sent = tx_map.get(&sent_txid).unwrap();
-                    assert_eq!(sent.received, details.received);
-                    assert_eq!(sent.sent, details.sent);
-                    assert_eq!(sent.fees, details.fees);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_long_change_chain() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let mut total_sent = 0;
-                    for _ in 0..5 {
-                        let mut builder = wallet.build_tx();
-                        builder.add_recipient(node_addr.script_pubkey(), 5_000);
-                        let (psbt, details) = builder.finish().unwrap();
-                        let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                        assert!(finalized, "Cannot finalize transaction");
-                        wallet.broadcast(psbt.extract_tx()).unwrap();
-
-                        wallet.sync(noop_progress(), None).unwrap();
-
-                        total_sent += 5_000 + details.fees;
-                    }
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
-
-                    // empty wallet
-                    let wallet = get_wallet_from_descriptors(&descriptors);
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let mut builder = wallet.build_tx();
-                    builder.add_recipient(node_addr.script_pubkey().clone(), 5_000).enable_rbf();
-                    let (psbt, details) = builder.finish().unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fees - 5_000);
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    let mut builder = wallet.build_fee_bump(details.txid).unwrap();
-                    builder.fee_rate(FeeRate::from_sat_per_vb(2.1));
-                    let (new_psbt, new_details) = builder.finish().unwrap();
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fees - 5_000);
-                    assert_eq!(wallet.get_balance().unwrap(), new_details.received);
-
-                    assert!(new_details.fees > details.fees);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee_remove_change() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let mut builder = wallet.build_tx();
-                    builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
-                    let (psbt, details) = builder.finish().unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fees);
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    let mut builder = wallet.build_fee_bump(details.txid).unwrap();
-                    builder.fee_rate(FeeRate::from_sat_per_vb(5.0));
-                    let (new_psbt, new_details) = builder.finish().unwrap();
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 0);
-                    assert_eq!(new_details.received, 0);
-
-                    assert!(new_details.fees > details.fees);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee_add_input() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-
-                    let mut builder = wallet.build_tx();
-                    builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
-                    let (psbt, details) = builder.finish().unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                    assert_eq!(details.received, 1_000 - details.fees);
-
-                    let mut builder = wallet.build_fee_bump(details.txid).unwrap();
-                    builder.fee_rate(FeeRate::from_sat_per_vb(10.0));
-                    let (new_psbt, new_details) = builder.finish().unwrap();
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(new_details.sent, 75_000);
-                    assert_eq!(wallet.get_balance().unwrap(), new_details.received);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee_add_input_no_change() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-
-                    let mut builder = wallet.build_tx();
-                    builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
-                    let (psbt, details) = builder.finish().unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                    assert_eq!(details.received, 1_000 - details.fees);
-
-                    let mut builder = wallet.build_fee_bump(details.txid).unwrap();
-                    builder.fee_rate(FeeRate::from_sat_per_vb(123.0));
-                    let (new_psbt, new_details) = builder.finish().unwrap();
-                    println!("{:#?}", new_details);
-
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(new_details.sent, 75_000);
-                    assert_eq!(wallet.get_balance().unwrap(), 0);
-                    assert_eq!(new_details.received, 0);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_receive_coinbase() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let wallet_addr = wallet.get_new_address().unwrap();
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 0);
-
-                    test_client.generate(1, Some(wallet_addr));
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert!(wallet.get_balance().unwrap() > 0);
-                }
-            }
-
-                        };
-
-            output.into()
-        }
-    }
-}

testutils/.gitignore

@@ -1,2 +0,0 @@
-target/
-Cargo.lock

testutils/Cargo.toml

@@ -1,26 +0,0 @@
-[package]
-name = "bdk-testutils"
-version = "0.3.0"
-authors = ["Alekos Filini <alekos.filini@gmail.com>"]
-edition = "2018"
-homepage = "https://bitcoindevkit.org"
-repository = "https://github.com/bitcoindevkit/bdk"
-documentation = "https://docs.rs/bdk-testutils"
-description = "Supporting testing utilities for `bdk`"
-keywords = ["bdk"]
-license = "MIT"
-
-[lib]
-name = "testutils"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-log = "0.4.8"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-serial_test = "0.4"
-bitcoin = "0.26"
-bitcoincore-rpc = "0.13"
-miniscript = "5.1"
-electrum-client = "0.6.0"

testutils/src/lib.rs

@@ -1,575 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-#[macro_use]
-extern crate serde_json;
-
-pub use serial_test::serial;
-
-use std::collections::HashMap;
-use std::env;
-use std::ops::Deref;
-use std::path::PathBuf;
-use std::str::FromStr;
-use std::time::Duration;
-
-#[allow(unused_imports)]
-use log::{debug, error, info, trace};
-
-use bitcoin::consensus::encode::{deserialize, serialize};
-use bitcoin::hashes::hex::{FromHex, ToHex};
-use bitcoin::hashes::sha256d;
-use bitcoin::secp256k1::{Secp256k1, Verification};
-use bitcoin::{Address, Amount, PublicKey, Script, Transaction, Txid};
-
-use miniscript::descriptor::DescriptorPublicKey;
-use miniscript::{Descriptor, MiniscriptKey, TranslatePk};
-
-pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
-pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
-
-pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
-
-// TODO: we currently only support env vars, we could also parse a toml file
-fn get_auth() -> Auth {
-    match env::var("MAGICAL_RPC_AUTH").as_ref().map(String::as_ref) {
-        Ok("USER_PASS") => Auth::UserPass(
-            env::var("MAGICAL_RPC_USER").unwrap(),
-            env::var("MAGICAL_RPC_PASS").unwrap(),
-        ),
-        _ => Auth::CookieFile(PathBuf::from(
-            env::var("MAGICAL_RPC_COOKIEFILE")
-                .unwrap_or("/home/user/.bitcoin/regtest/.cookie".to_string()),
-        )),
-    }
-}
-
-pub fn get_electrum_url() -> String {
-    env::var("MAGICAL_ELECTRUM_URL").unwrap_or("tcp://127.0.0.1:50001".to_string())
-}
-
-pub struct TestClient {
-    client: RpcClient,
-    electrum: ElectrumClient,
-}
-
-#[derive(Clone, Debug)]
-pub struct TestIncomingOutput {
-    pub value: u64,
-    pub to_address: String,
-}
-
-impl TestIncomingOutput {
-    pub fn new(value: u64, to_address: Address) -> Self {
-        Self {
-            value,
-            to_address: to_address.to_string(),
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct TestIncomingTx {
-    pub output: Vec<TestIncomingOutput>,
-    pub min_confirmations: Option<u64>,
-    pub locktime: Option<i64>,
-    pub replaceable: Option<bool>,
-}
-
-impl TestIncomingTx {
-    pub fn new(
-        output: Vec<TestIncomingOutput>,
-        min_confirmations: Option<u64>,
-        locktime: Option<i64>,
-        replaceable: Option<bool>,
-    ) -> Self {
-        Self {
-            output,
-            min_confirmations,
-            locktime,
-            replaceable,
-        }
-    }
-
-    pub fn add_output(&mut self, output: TestIncomingOutput) {
-        self.output.push(output);
-    }
-}
-
-#[doc(hidden)]
-pub trait TranslateDescriptor {
-    // derive and translate a `Descriptor<DescriptorPublicKey>` into a `Descriptor<PublicKey>`
-    fn derive_translated<C: Verification>(
-        &self,
-        secp: &Secp256k1<C>,
-        index: u32,
-    ) -> Descriptor<PublicKey>;
-}
-
-impl TranslateDescriptor for Descriptor<DescriptorPublicKey> {
-    fn derive_translated<C: Verification>(
-        &self,
-        secp: &Secp256k1<C>,
-        index: u32,
-    ) -> Descriptor<PublicKey> {
-        let translate = |key: &DescriptorPublicKey| -> PublicKey {
-            match key {
-                DescriptorPublicKey::XPub(xpub) => {
-                    xpub.xkey
-                        .derive_pub(secp, &xpub.derivation_path)
-                        .expect("hardened derivation steps")
-                        .public_key
-                }
-                DescriptorPublicKey::SinglePub(key) => key.key,
-            }
-        };
-
-        self.derive(index)
-            .translate_pk_infallible(|pk| translate(pk), |pkh| translate(pkh).to_pubkeyhash())
-    }
-}
-
-#[macro_export]
-macro_rules! testutils {
-    ( @external $descriptors:expr, $child:expr ) => ({
-        use bitcoin::secp256k1::Secp256k1;
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
-
-        use $crate::TranslateDescriptor;
-
-        let secp = Secp256k1::new();
-
-        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &$descriptors.0).expect("Failed to parse descriptor in `testutils!(@external)`").0;
-        parsed.derive_translated(&secp, $child).address(bitcoin::Network::Regtest).expect("No address form")
-    });
-    ( @internal $descriptors:expr, $child:expr ) => ({
-        use bitcoin::secp256k1::Secp256k1;
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
-
-        use $crate::TranslateDescriptor;
-
-        let secp = Secp256k1::new();
-
-        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
-        parsed.derive_translated(&secp, $child).address(bitcoin::Network::Regtest).expect("No address form")
-    });
-    ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
-    ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
-
-    ( @tx ( $( ( $( $addr:tt )* ) => $amount:expr ),+ ) $( ( @locktime $locktime:expr ) )* $( ( @confirmations $confirmations:expr ) )* $( ( @replaceable $replaceable:expr ) )* ) => ({
-        let mut outs = Vec::new();
-        $( outs.push(testutils::TestIncomingOutput::new($amount, testutils!( $($addr)* ))); )+
-
-        let mut locktime = None::<i64>;
-        $( locktime = Some($locktime); )*
-
-        let mut min_confirmations = None::<u64>;
-        $( min_confirmations = Some($confirmations); )*
-
-        let mut replaceable = None::<bool>;
-        $( replaceable = Some($replaceable); )*
-
-        testutils::TestIncomingTx::new(outs, min_confirmations, locktime, replaceable)
-    });
-
-    ( @literal $key:expr ) => ({
-        let key = $key.to_string();
-        (key, None::<String>, None::<String>)
-    });
-    ( @generate_xprv $( $external_path:expr )* $( ,$internal_path:expr )* ) => ({
-        use rand::Rng;
-
-        let mut seed = [0u8; 32];
-        rand::thread_rng().fill(&mut seed[..]);
-
-        let key = bitcoin::util::bip32::ExtendedPrivKey::new_master(
-            bitcoin::Network::Testnet,
-            &seed,
-        );
-
-        let mut external_path = None::<String>;
-        $( external_path = Some($external_path.to_string()); )*
-
-        let mut internal_path = None::<String>;
-        $( internal_path = Some($internal_path.to_string()); )*
-
-        (key.unwrap().to_string(), external_path, internal_path)
-    });
-    ( @generate_wif ) => ({
-        use rand::Rng;
-
-        let mut key = [0u8; bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
-        rand::thread_rng().fill(&mut key[..]);
-
-        (bitcoin::PrivateKey {
-            compressed: true,
-            network: bitcoin::Network::Testnet,
-            key: bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
-        }.to_string(), None::<String>, None::<String>)
-    });
-
-    ( @keys ( $( $alias:expr => ( $( $key_type:tt )* ) ),+ ) ) => ({
-        let mut map = std::collections::HashMap::new();
-        $(
-            let alias: &str = $alias;
-            map.insert(alias, testutils!( $($key_type)* ));
-        )+
-
-        map
-    });
-
-    ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )* $( ( @keys $( $keys:tt )* ) )* ) => ({
-        use std::str::FromStr;
-        use std::collections::HashMap;
-        use std::convert::TryInto;
-
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
-        use miniscript::TranslatePk;
-
-        let mut keys: HashMap<&'static str, (String, Option<String>, Option<String>)> = HashMap::new();
-        $(
-            keys = testutils!{ @keys $( $keys )* };
-        )*
-
-        let external: Descriptor<String> = FromStr::from_str($external_descriptor).unwrap();
-        let external: Descriptor<String> = external.translate_pk_infallible::<_, _>(|k| {
-            if let Some((key, ext_path, _)) = keys.get(&k.as_str()) {
-                format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))
-            } else {
-                k.clone()
-            }
-        }, |kh| {
-            if let Some((key, ext_path, _)) = keys.get(&kh.as_str()) {
-                format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))
-            } else {
-                kh.clone()
-            }
-
-        });
-        let external = external.to_string();
-
-        let mut internal = None::<String>;
-        $(
-            let string_internal: Descriptor<String> = FromStr::from_str($internal_descriptor).unwrap();
-
-            let string_internal: Descriptor<String> = string_internal.translate_pk_infallible::<_, _>(|k| {
-                if let Some((key, _, int_path)) = keys.get(&k.as_str()) {
-                    format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))
-                } else {
-                    k.clone()
-                }
-            }, |kh| {
-                if let Some((key, _, int_path)) = keys.get(&kh.as_str()) {
-                    format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))
-                } else {
-                    kh.clone()
-                }
-            });
-            internal = Some(string_internal.to_string());
-        )*
-
-        (external, internal)
-    })
-}
-
-fn exponential_backoff_poll<T, F>(mut poll: F) -> T
-where
-    F: FnMut() -> Option<T>,
-{
-    let mut delay = Duration::from_millis(64);
-    loop {
-        match poll() {
-            Some(data) => break data,
-            None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0),
-            None => {}
-        }
-
-        std::thread::sleep(delay);
-    }
-}
-
-impl TestClient {
-    pub fn new() -> Self {
-        let url = env::var("MAGICAL_RPC_URL").unwrap_or("127.0.0.1:18443".to_string());
-        let client = RpcClient::new(format!("http://{}", url), get_auth()).unwrap();
-        let electrum = ElectrumClient::new(&get_electrum_url()).unwrap();
-
-        TestClient { client, electrum }
-    }
-
-    fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
-        // wait for electrs to index the tx
-        exponential_backoff_poll(|| {
-            trace!("wait_for_tx {}", txid);
-
-            self.electrum
-                .script_get_history(monitor_script)
-                .unwrap()
-                .iter()
-                .position(|entry| entry.tx_hash == txid)
-        });
-    }
-
-    fn wait_for_block(&mut self, min_height: usize) {
-        self.electrum.block_headers_subscribe().unwrap();
-
-        loop {
-            let header = exponential_backoff_poll(|| {
-                self.electrum.ping().unwrap();
-                self.electrum.block_headers_pop().unwrap()
-            });
-            if header.height >= min_height {
-                break;
-            }
-        }
-    }
-
-    pub fn receive(&mut self, meta_tx: TestIncomingTx) -> Txid {
-        assert!(
-            meta_tx.output.len() > 0,
-            "can't create a transaction with no outputs"
-        );
-
-        let mut map = HashMap::new();
-
-        let mut required_balance = 0;
-        for out in &meta_tx.output {
-            required_balance += out.value;
-            map.insert(out.to_address.clone(), Amount::from_sat(out.value));
-        }
-
-        if self.get_balance(None, None).unwrap() < Amount::from_sat(required_balance) {
-            panic!("Insufficient funds in bitcoind. Plase generate a few blocks with: `bitcoin-cli generatetoaddress 10 {}`", self.get_new_address(None, None).unwrap());
-        }
-
-        // FIXME: core can't create a tx with two outputs to the same address
-        let tx = self
-            .create_raw_transaction_hex(&[], &map, meta_tx.locktime, meta_tx.replaceable)
-            .unwrap();
-        let tx = self.fund_raw_transaction(tx, None, None).unwrap();
-        let mut tx: Transaction = deserialize(&tx.hex).unwrap();
-
-        if let Some(true) = meta_tx.replaceable {
-            // for some reason core doesn't set this field right
-            for input in &mut tx.input {
-                input.sequence = 0xFFFFFFFD;
-            }
-        }
-
-        let tx = self
-            .sign_raw_transaction_with_wallet(&serialize(&tx), None, None)
-            .unwrap();
-
-        // broadcast through electrum so that it caches the tx immediately
-        let txid = self
-            .electrum
-            .transaction_broadcast(&deserialize(&tx.hex).unwrap())
-            .unwrap();
-
-        if let Some(num) = meta_tx.min_confirmations {
-            self.generate(num, None);
-        }
-
-        let monitor_script = Address::from_str(&meta_tx.output[0].to_address)
-            .unwrap()
-            .script_pubkey();
-        self.wait_for_tx(txid, &monitor_script);
-
-        debug!("Sent tx: {}", txid);
-
-        txid
-    }
-
-    pub fn bump_fee(&mut self, txid: &Txid) -> Txid {
-        let tx = self.get_raw_transaction_info(txid, None).unwrap();
-        assert!(
-            tx.confirmations.is_none(),
-            "Can't bump tx {} because it's already confirmed",
-            txid
-        );
-
-        let bumped: serde_json::Value = self.call("bumpfee", &[txid.to_string().into()]).unwrap();
-        let new_txid = Txid::from_str(&bumped["txid"].as_str().unwrap().to_string()).unwrap();
-
-        let monitor_script =
-            tx.vout[0].script_pub_key.addresses.as_ref().unwrap()[0].script_pubkey();
-        self.wait_for_tx(new_txid, &monitor_script);
-
-        debug!("Bumped {}, new txid {}", txid, new_txid);
-
-        new_txid
-    }
-
-    pub fn generate_manually(&mut self, txs: Vec<Transaction>) -> String {
-        use bitcoin::blockdata::block::{Block, BlockHeader};
-        use bitcoin::blockdata::script::Builder;
-        use bitcoin::blockdata::transaction::{OutPoint, TxIn, TxOut};
-        use bitcoin::hash_types::{BlockHash, TxMerkleNode};
-
-        let block_template: serde_json::Value = self
-            .call("getblocktemplate", &[json!({"rules": ["segwit"]})])
-            .unwrap();
-        trace!("getblocktemplate: {:#?}", block_template);
-
-        let header = BlockHeader {
-            version: block_template["version"].as_i64().unwrap() as i32,
-            prev_blockhash: BlockHash::from_hex(
-                block_template["previousblockhash"].as_str().unwrap(),
-            )
-            .unwrap(),
-            merkle_root: TxMerkleNode::default(),
-            time: block_template["curtime"].as_u64().unwrap() as u32,
-            bits: u32::from_str_radix(block_template["bits"].as_str().unwrap(), 16).unwrap(),
-            nonce: 0,
-        };
-        debug!("header: {:#?}", header);
-
-        let height = block_template["height"].as_u64().unwrap() as i64;
-        let witness_reserved_value: Vec<u8> = sha256d::Hash::default().as_ref().into();
-        // burn block subsidy and fees, not a big deal
-        let mut coinbase_tx = Transaction {
-            version: 1,
-            lock_time: 0,
-            input: vec![TxIn {
-                previous_output: OutPoint::null(),
-                script_sig: Builder::new().push_int(height).into_script(),
-                sequence: 0xFFFFFFFF,
-                witness: vec![witness_reserved_value],
-            }],
-            output: vec![],
-        };
-
-        let mut txdata = vec![coinbase_tx.clone()];
-        txdata.extend_from_slice(&txs);
-
-        let mut block = Block { header, txdata };
-
-        let witness_root = block.witness_root();
-        let witness_commitment =
-            Block::compute_witness_commitment(&witness_root, &coinbase_tx.input[0].witness[0]);
-
-        // now update and replace the coinbase tx
-        let mut coinbase_witness_commitment_script = vec![0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed];
-        coinbase_witness_commitment_script.extend_from_slice(&witness_commitment);
-
-        coinbase_tx.output.push(TxOut {
-            value: 0,
-            script_pubkey: coinbase_witness_commitment_script.into(),
-        });
-        block.txdata[0] = coinbase_tx;
-
-        // set merkle root
-        let merkle_root = block.merkle_root();
-        block.header.merkle_root = merkle_root;
-
-        assert!(block.check_merkle_root());
-        assert!(block.check_witness_commitment());
-
-        // now do PoW :)
-        let target = block.header.target();
-        while block.header.validate_pow(&target).is_err() {
-            block.header.nonce = block.header.nonce.checked_add(1).unwrap(); // panic if we run out of nonces
-        }
-
-        let block_hex: String = serialize(&block).to_hex();
-        debug!("generated block hex: {}", block_hex);
-
-        self.electrum.block_headers_subscribe().unwrap();
-
-        let submit_result: serde_json::Value =
-            self.call("submitblock", &[block_hex.into()]).unwrap();
-        debug!("submitblock: {:?}", submit_result);
-        assert!(
-            submit_result.is_null(),
-            "submitblock error: {:?}",
-            submit_result.as_str()
-        );
-
-        self.wait_for_block(height as usize);
-
-        block.header.block_hash().to_hex()
-    }
-
-    pub fn generate(&mut self, num_blocks: u64, address: Option<Address>) {
-        let address = address.unwrap_or_else(|| self.get_new_address(None, None).unwrap());
-        let hashes = self.generate_to_address(num_blocks, &address).unwrap();
-        let best_hash = hashes.last().unwrap();
-        let height = self.get_block_info(best_hash).unwrap().height;
-
-        self.wait_for_block(height);
-
-        debug!("Generated blocks to new height {}", height);
-    }
-
-    pub fn invalidate(&mut self, num_blocks: u64) {
-        self.electrum.block_headers_subscribe().unwrap();
-
-        let best_hash = self.get_best_block_hash().unwrap();
-        let initial_height = self.get_block_info(&best_hash).unwrap().height;
-
-        let mut to_invalidate = best_hash;
-        for i in 1..=num_blocks {
-            trace!(
-                "Invalidating block {}/{} ({})",
-                i,
-                num_blocks,
-                to_invalidate
-            );
-
-            self.invalidate_block(&to_invalidate).unwrap();
-            to_invalidate = self.get_best_block_hash().unwrap();
-        }
-
-        self.wait_for_block(initial_height - num_blocks as usize);
-
-        debug!(
-            "Invalidated {} blocks to new height of {}",
-            num_blocks,
-            initial_height - num_blocks as usize
-        );
-    }
-
-    pub fn reorg(&mut self, num_blocks: u64) {
-        self.invalidate(num_blocks);
-        self.generate(num_blocks, None);
-    }
-
-    pub fn get_node_address(&self, address_type: Option<AddressType>) -> Address {
-        Address::from_str(
-            &self
-                .get_new_address(None, address_type)
-                .unwrap()
-                .to_string(),
-        )
-        .unwrap()
-    }
-}
-
-impl Deref for TestClient {
-    type Target = RpcClient;
-
-    fn deref(&self) -> &Self::Target {
-        &self.client
-    }
-}