Bump version to 0.9.1-dev

Verify the unconfirmed transactions we download against the consensus
rules. This is currently exposed as an extra `verify` feature, since it
depends on a pre-release version of `bitcoinconsensus`.

Closes #352

.github/pull_request_template.md

@@ -0,0 +1,30 @@
+<!-- You can erase any parts of this template not applicable to your Pull Request. -->
+
+### Description
+
+<!-- Describe the purpose of this PR, what's being adding and/or fixed -->
+
+### Notes to the reviewers
+
+<!-- In this section you can include notes directed to the reviewers, like explaining why some parts
+of the PR were done in a specific way -->
+
+### Checklists
+
+#### All Submissions:
+
+* [ ] I've signed all my commits
+* [ ] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
+* [ ] I ran `cargo fmt` and `cargo clippy` before committing
+
+#### New Features:
+
+* [ ] I've added tests for the new feature
+* [ ] I've added docs for the new feature
+* [ ] I've updated `CHANGELOG.md`
+
+#### Bugfixes:
+
+* [ ] This pull request breaks the existing API
+* [ ] I've added tests to reproduce the issue which are now passing
+* [ ] I'm linking the issue being fixed by this PR

.github/workflows/audit.yml

@@ -0,0 +1,19 @@
+name: Audit
+
+on:
+  push:
+    paths:
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+  schedule:
+    - cron: '0 0 * * 0' # Once per week
+
+jobs:
+
+  security_audit:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/code_coverage.yml

@@ -0,0 +1,37 @@
+on: [push]
+
+name: Code Coverage
+
+jobs:
+
+  Codecov:
+    name: Code Coverage
+    runs-on: ubuntu-latest
+    env:
+      CARGO_INCREMENTAL: '0'
+      RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off'
+      RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Install rustup
+        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Update toolchain
+        run: rustup update
+      - name: Test
+        run: cargo test --features all-keys,compiler,esplora,compact_filters --no-default-features
+                    
+      - id: coverage
+        name: Generate coverage
+        uses: actions-rs/grcov@v0.1.5
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          file: ${{ steps.coverage.outputs.report }}
+          directory: ./coverage/reports/

.github/workflows/cont_integration.yml

@@ -0,0 +1,178 @@
+on: [push, pull_request]
+
+name: CI
+
+jobs:
+
+  build-test:
+    name: Build and test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - 1.53.0 # STABLE
+          - 1.46.0 # MSRV
+        features:
+          - default
+          - minimal
+          - all-keys
+          - minimal,esplora
+          - key-value-db
+          - electrum
+          - compact_filters
+          - esplora,key-value-db,electrum
+          - compiler
+          - rpc
+          - verify
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Generate cache key
+        run: echo "${{ matrix.rust }} ${{ matrix.features }}" | tee .cache_key
+      - name: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('.cache_key') }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Set default toolchain
+        run: rustup default ${{ matrix.rust }}
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add clippy
+        run: rustup component add clippy
+      - name: Update toolchain
+        run: rustup update
+      - name: Build
+        run: cargo build --features ${{ matrix.features }} --no-default-features
+      - name: Clippy
+        run: cargo clippy --all-targets --features ${{ matrix.features }} --no-default-features -- -D warnings
+      - name: Test
+        run: cargo test --features ${{ matrix.features }} --no-default-features
+
+  test-readme-examples:
+    name: Test README.md examples
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-test-md-docs-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Update toolchain
+        run: rustup update
+      - name: Test
+        run: cargo test --features test-md-docs --no-default-features -- doctest::ReadmeDoctests
+
+  test-blockchains:
+    name: Test ${{ matrix.blockchain.name }}
+    runs-on: ubuntu-16.04
+    strategy:
+      fail-fast: false
+      matrix:
+        blockchain:
+          - name: electrum
+            container: bitcoindevkit/electrs:0.4.0
+            start: /root/electrs --network regtest --cookie-file $GITHUB_WORKSPACE/.bitcoin/regtest/.cookie --jsonrpc-import
+          - name: esplora
+            container: bitcoindevkit/esplora:0.4.0
+            start: /root/electrs --network regtest -vvv --daemon-dir $GITHUB_WORKSPACE/.bitcoin --jsonrpc-import --electrum-rpc-addr=0.0.0.0:60401 --http-addr 0.0.0.0:3002
+          - name: rpc
+            container: bitcoindevkit/electrs:0.4.0
+            start: /root/electrs --network regtest --cookie-file $GITHUB_WORKSPACE/.bitcoin/regtest/.cookie --jsonrpc-import
+    container: ${{ matrix.blockchain.container }}
+    env:
+      BDK_RPC_AUTH: COOKIEFILE
+      BDK_RPC_COOKIEFILE: ${{ github.workspace }}/.bitcoin/regtest/.cookie
+      BDK_RPC_URL: 127.0.0.1:18443
+      BDK_RPC_WALLET: bdk-test
+      BDK_ELECTRUM_URL: tcp://127.0.0.1:60401
+      BDK_ESPLORA_URL: http://127.0.0.1:3002
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: get pkg-config # running esplora tests seems to need this
+        run: apt update && apt install -y --fix-missing pkg-config libssl-dev
+      - name: Install rustup
+        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - name: Set default toolchain
+        run: $HOME/.cargo/bin/rustup default 1.53.0 # STABLE
+      - name: Set profile
+        run: $HOME/.cargo/bin/rustup set profile minimal
+      - name: Update toolchain
+        run: $HOME/.cargo/bin/rustup update
+      - name: Start core
+        run: ./ci/start-core.sh
+      - name: start ${{ matrix.blockchain.name }}
+        run: nohup ${{ matrix.blockchain.start }} & sleep 5
+      - name: Test
+        run: $HOME/.cargo/bin/cargo test --features test-${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests
+
+  check-wasm:
+    name: Check WASM
+    runs-on: ubuntu-16.04
+    env:
+      CC: clang-10
+      CFLAGS: -I/usr/include
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+        # Install a recent version of clang that supports wasm32
+      - run: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
+      - run: sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" || exit 1
+      - run: sudo apt-get update || exit 1
+      - run: sudo apt-get install -y libclang-common-10-dev clang-10 libc6-dev-i386 || exit 1
+      - name: Set default toolchain
+        run: rustup default 1.53.0 # STABLE
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add target wasm32
+        run: rustup target add wasm32-unknown-unknown
+      - name: Update toolchain
+        run: rustup update
+      - name: Check
+        run: cargo check --target wasm32-unknown-unknown --features esplora --no-default-features
+
+  fmt:
+    name: Rust fmt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add rustfmt
+        run: rustup component add rustfmt
+      - name: Update toolchain
+        run: rustup update
+      - name: Check fmt
+        run: cargo fmt --all -- --config format_code_in_doc_comments=true --check

.github/workflows/nightly_docs.yml

@@ -0,0 +1,61 @@
+name: Publish Nightly Docs
+
+on: [push, pull_request]
+
+jobs:
+  build_docs:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Setup cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: nightly-docs-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Update toolchain
+        run: rustup update
+      - name: Build docs
+        run: cargo rustdoc --verbose --features=compiler,electrum,esplora,compact_filters,key-value-db,all-keys -- --cfg docsrs -Dwarnings
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: built-docs
+          path: ./target/doc/*
+
+  publish_docs:
+    name: 'Publish docs'
+    if: github.ref == 'refs/heads/master'
+    needs: [build_docs]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout `bitcoindevkit.org`
+        uses: actions/checkout@v2
+        with:
+          ssh-key: ${{ secrets.DOCS_PUSH_SSH_KEY }}
+          repository: bitcoindevkit/bitcoindevkit.org
+          ref: master
+      - name: Create directories
+        run: mkdir -p ./static/docs-rs/bdk/nightly
+      - name: Remove old latest
+        run: rm -rf ./static/docs-rs/bdk/nightly/latest
+      - name: Download built docs
+        uses: actions/download-artifact@v1
+        with:
+          name: built-docs
+          path: ./static/docs-rs/bdk/nightly/latest
+      - name: Configure git
+        run: git config user.email "github-actions@github.com" && git config user.name "github-actions"
+      - name: Commit
+        continue-on-error: true         # If there's nothing to commit this step fails, but it's fine
+        run: git add ./static && git commit -m "Publish autogenerated nightly docs"
+      - name: Push
+        run: git push origin master

.gitignore

@@ -2,3 +2,4 @@
 Cargo.lock
 
 *.swp
+.idea

.travis.yml

@@ -1,65 +0,0 @@
-language: rust
-rust:
-  - stable
-
-env:
-  global:
-    - MAGICAL_RPC_COOKIEFILE=/home/travis/.bitcoin/regtest/.cookie
-    - MAGICAL_ELECTRUM_URL=tcp://127.0.0.1:60401
-  jobs:
-    - TARGET=x86_64-unknown-linux-gnu    CHECK_FMT=1
-    - TARGET=x86_64-unknown-linux-gnu    RUN_TESTS=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=minimal            NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=minimal,esplora    NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=key-value-db       NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=electrum           NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=compact_filters    NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=cli-utils,esplora  NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=compiler           NO_DEFAULT_FEATURES=1    RUN_TESTS=1 # Test the `miniscriptc` example
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=test-electrum      NO_DEFAULT_FEATURES=1    RUN_TESTS=1    RUN_CORE=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=test-md-docs       NO_DEFAULT_FEATURES=1    RUN_TESTS=1    NIGHTLY=1
-    - TARGET=wasm32-unknown-unknown      FEATURES=cli-utils,esplora  NO_DEFAULT_FEATURES=1
-before_script:
-  - |
-    if [[ "$TARGET" = "wasm32-unknown-unknown" ]]; then
-      # Install a recent version of clang that supports wasm32
-      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
-      sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" || exit 1
-      sudo apt-get update || exit 1
-      sudo apt-get install -y clang-10 libc6-dev-i386 || exit 1
-      export CC="clang-10"
-      export CFLAGS="-I/usr/include"
-    fi
-  - |
-    if [[ $CHECK_FMT -eq 1 ]]; then
-      rustup component add rustfmt
-    fi
-  - |
-    if [[ $NIGHTLY -eq 1 ]]; then
-      rustup toolchain install nightly
-      rustup default nightly
-    fi
-  - rustup target add "$TARGET"
-script:
-  - |
-    if [[ $CHECK_FMT -eq 1 ]]; then
-      cargo fmt -- --check || exit 1
-    fi
-  - |
-    if [[ $RUN_TESTS -eq 1 ]]; then
-        CMD=test
-    else
-        CMD=build
-    fi
-  - |
-    if [[ $RUN_CORE -eq 1 ]]; then
-        ./ci/start-core.sh || exit 1
-    fi
-  - cargo $CMD --verbose --target=$TARGET --features=$FEATURES $( (( NO_DEFAULT_FEATURES == 1 )) && printf %s '--no-default-features' )
-
-notifications:
-  email: false
-
-before_cache:
-  - rm -rf "$TRAVIS_HOME/.cargo/registry/src"
-cache: cargo

CHANGELOG.md

@@ -0,0 +1,359 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [v0.9.0] - [v0.8.0]
+
+### Wallet
+#### Added
+- Bitcoin core RPC added as blockchain backend
+- Add a `verify` feature that can be enable to verify the unconfirmed txs we download against the consensus rules
+
+## [v0.8.0] - [v0.7.0]
+
+### Wallet
+- Added an option that must be explicitly enabled to allow signing using non-`SIGHASH_ALL` sighashes (#350)
+#### Changed
+`get_address` now returns an `AddressInfo` struct that includes the index and derefs to `Address`.
+
+## [v0.7.0] - [v0.6.0]
+
+### Policy
+#### Changed
+Removed `fill_satisfaction` method in favor of enum parameter in `extract_policy` method
+
+#### Added
+Timelocks are considered (optionally) in building the `satisfaction` field
+
+### Wallet
+
+- Changed `Wallet::{sign, finalize_psbt}` now take a `&mut psbt` rather than consuming it.
+- Require and validate `non_witness_utxo` for SegWit signatures by default, can be adjusted with `SignOptions`
+- Replace the opt-in builder option `force_non_witness_utxo` with the opposite `only_witness_utxo`. From now on we will provide the `non_witness_utxo`, unless explicitly asked not to.
+
+## [v0.6.0] - [v0.5.1]
+
+### Misc
+#### Changed
+- New minimum supported rust version is 1.46.0
+- Changed `AnyBlockchainConfig` to use serde tagged representation.
+
+### Descriptor
+#### Added
+- Added ability to analyze a `PSBT` to check which and how many signatures are already available
+
+### Wallet
+#### Changed
+- `get_new_address()` refactored to `get_address(AddressIndex::New)` to support different `get_address()` index selection strategies
+
+#### Added
+- Added `get_address(AddressIndex::LastUnused)` which returns the last derived address if it has not been used or if used in a received transaction returns a new address
+- Added `get_address(AddressIndex::Peek(u32))` which returns a derived address for a specified descriptor index but does not change the current index
+- Added `get_address(AddressIndex::Reset(u32))` which returns a derived address for a specified descriptor index and resets current index to the given value
+- Added `get_psbt_input` to create the corresponding psbt input for a local utxo.
+
+#### Fixed
+- Fixed `coin_select` calculation for UTXOs where `value < fee` that caused over-/underflow errors.
+
+## [v0.5.1] - [v0.5.0]
+
+### Misc
+#### Changed
+- Pin `hyper` to `=0.14.4` to make it compile on Rust 1.45
+
+## [v0.5.0] - [v0.4.0]
+
+### Misc
+#### Changed
+- Updated `electrum-client` to version `0.7`
+
+### Wallet
+#### Changed
+- `FeeRate` constructors `from_sat_per_vb` and `default_min_relay_fee` are now `const` functions
+
+## [v0.4.0] - [v0.3.0]
+
+### Keys
+#### Changed
+- Renamed `DerivableKey::add_metadata()` to `DerivableKey::into_descriptor_key()`
+- Renamed `ToDescriptorKey::to_descriptor_key()` to `IntoDescriptorKey::into_descriptor_key()`
+#### Added
+- Added an `ExtendedKey` type that is an enum of `bip32::ExtendedPubKey` and `bip32::ExtendedPrivKey`
+- Added `DerivableKey::into_extended_key()` as the only method that needs to be implemented
+
+### Misc
+#### Removed
+- Removed the `parse_descriptor` example, since it wasn't demonstrating any bdk-specific API anymore.
+#### Changed
+- Updated `bitcoin` to `0.26`, `miniscript` to `5.1` and `electrum-client` to `0.6`
+#### Added
+- Added support for the `signet` network (issue #62)
+- Added a function to get the version of BDK at runtime
+
+### Wallet
+#### Changed
+- Removed the explicit `id` argument from `Wallet::add_signer()` since that's now part of `Signer` itself
+- Renamed `ToWalletDescriptor::to_wallet_descriptor()` to `IntoWalletDescriptor::into_wallet_descriptor()`
+
+### Policy
+#### Changed
+- Removed unneeded `Result<(), PolicyError>` return type for `Satisfaction::finalize()`
+- Removed the `TooManyItemsSelected` policy error (see commit message for more details)
+
+## [v0.3.0] - [v0.2.0]
+
+### Descriptor
+#### Changed
+- Added an alias `DescriptorError` for `descriptor::error::Error`
+- Changed the error returned by `descriptor!()` and `fragment!()` to `DescriptorError`
+- Changed the error type in `ToWalletDescriptor` to `DescriptorError`
+- Improved checks on descriptors built using the macros
+
+### Blockchain
+#### Changed
+- Remove `BlockchainMarker`, `OfflineClient` and `OfflineWallet` in favor of just using the unit
+  type to mark for a missing client.
+- Upgrade `tokio` to `1.0`.
+
+### Transaction Creation Overhaul
+
+The `TxBuilder` is now created from the `build_tx` or `build_fee_bump` functions on wallet and the
+final transaction is created by calling `finish` on the builder.
+
+- Removed `TxBuilder::utxos` in favor of `TxBuilder::add_utxos`
+- Added `Wallet::build_tx` to replace `Wallet::create_tx`
+- Added `Wallet::build_fee_bump` to replace `Wallet::bump_fee`
+- Added `Wallet::get_utxo`
+- Added `Wallet::get_descriptor_for_keychain`
+
+### `add_foreign_utxo`
+
+- Renamed `UTXO` to `LocalUtxo`
+- Added `WeightedUtxo` to replace floating `(UTXO, usize)`.
+- Added `Utxo` enum to incorporate both local utxos and foreign utxos
+- Added `TxBuilder::add_foreign_utxo` which allows adding a utxo external to the wallet.
+
+### CLI
+#### Changed
+- Remove `cli.rs` module, `cli-utils` feature and `repl.rs` example; moved to new [`bdk-cli`](https://github.com/bitcoindevkit/bdk-cli) repository
+
+## [v0.2.0] - [0.1.0-beta.1]
+
+### Project
+#### Added
+- Add CONTRIBUTING.md
+- Add a Discord badge to the README
+- Add code coverage github actions workflow
+- Add scheduled audit check in CI
+- Add CHANGELOG.md
+
+#### Changed
+- Rename the library to `bdk`
+- Rename `ScriptType` to `KeychainKind`
+- Prettify README examples on github
+- Change CI to github actions
+- Bump rust-bitcoin to 0.25, fix Cargo dependencies
+- Enable clippy for stable and tests by default
+- Switch to "mainline" rust-miniscript
+- Generate a different cache key for every CI job
+- Fix to at least bitcoin ^0.25.2
+
+#### Fixed
+- Fix or ignore clippy warnings for all optional features except compact_filters
+- Pin cc version because last breaks rocksdb build
+
+### Blockchain
+#### Added
+- Add a trait to create `Blockchain`s from a configuration
+- Add an `AnyBlockchain` enum to allow switching at runtime
+- Document `AnyBlockchain` and `ConfigurableBlockchain`
+- Use our Instant struct to be compatible with wasm
+- Make esplora call in parallel
+- Allow to set concurrency in Esplora config and optionally pass it in repl
+
+#### Fixed
+- Fix receiving a coinbase using Electrum/Esplora
+- Use proper type for EsploraHeader, make conversion to BlockHeader infallible
+- Eagerly unwrap height option, save one collect
+
+#### Changed
+- Simplify the architecture of blockchain traits
+- Improve sync
+- Remove unused varaint HeaderParseFail
+
+### CLI
+#### Added
+- Conditionally remove cli args according to enabled feature
+
+#### Changed
+- Add max_addresses param in sync
+- Split the internal and external policy paths
+
+### Database
+#### Added
+- Add `AnyDatabase` and `ConfigurableDatabase` traits
+
+### Descriptor
+#### Added
+- Add a macro to write descriptors from code
+- Add descriptor templates, add `DerivableKey`
+- Add ToWalletDescriptor trait tests
+- Add support for `sortedmulti` in `descriptor!`
+- Add ExtractPolicy trait tests
+- Add get_checksum tests, cleanup tests
+- Add descriptor macro tests
+
+#### Changes
+- Improve the descriptor macro, add traits for key and descriptor types
+
+#### Fixes
+- Fix the recovery of a descriptor given a PSBT
+
+### Keys
+#### Added
+- Add BIP39 support
+- Take `ScriptContext` into account when converting keys
+- Add a way to restrict the networks in which keys are valid
+- Add a trait for keys that can be generated
+- Fix entropy generation
+- Less convoluted entropy generation
+- Re-export tiny-bip39
+- Implement `GeneratableKey` trait for `bitcoin::PrivateKey`
+- Implement `ToDescriptorKey` trait for `GeneratedKey`
+- Add a shortcut to generate keys with the default options
+
+#### Fixed
+- Fix all-keys and cli-utils tests
+
+### Wallet
+#### Added
+- Allow to define static fees for transactions Fixes #137
+- Merging two match expressions for fee calculation
+- Incorporate RBF rules into utxo selection function
+- Add Branch and Bound coin selection
+- Add tests for BranchAndBoundCoinSelection::coin_select
+- Add tests for BranchAndBoundCoinSelection::bnb
+- Add tests for BranchAndBoundCoinSelection::single_random_draw
+- Add test that shwpkh populates witness_utxo
+- Add witness and redeem scripts to PSBT outputs
+- Add an option to include `PSBT_GLOBAL_XPUB`s in PSBTs
+- Eagerly finalize inputs
+
+#### Changed
+- Use collect to avoid iter unwrapping Options
+- Make coin_select take may/must use utxo lists
+- Improve `CoinSelectionAlgorithm`
+- Refactor `Wallet::bump_fee()`
+- Default to SIGHASH_ALL if not specified
+- Replace ChangeSpendPolicy::filter_utxos with a predicate
+- Make 'unspendable' into a HashSet
+- Stop implicitly enforcing manaul selection by .add_utxo
+- Rename DumbCS to LargestFirstCoinSelection
+- Rename must_use_utxos to required_utxos
+- Rename may_use_utxos to optional_uxtos
+- Rename get_must_may_use_utxos to preselect_utxos
+- Remove redundant Box around address validators
+- Remove redundant Box around signers
+- Make Signer and AddressValidator Send and Sync
+- Split `send_all` into `set_single_recipient` and `drain_wallet`
+- Use TXIN_DEFAULT_WEIGHT constant in coin selection
+- Replace `must_use` with `required` in coin selection
+- Take both spending policies into account in create_tx
+- Check last derivation in cache to avoid recomputation
+- Use the branch-and-bound cs by default
+- Make coin_select return UTXOs instead of TxIns
+- Build output lookup inside complete transaction
+- Don't wrap SignersContainer arguments in Arc
+- More consistent references with 'signers' variables
+
+#### Fixed
+- Fix signing for `ShWpkh` inputs
+- Fix the recovery of a descriptor given a PSBT
+
+### Examples
+#### Added
+- Support esplora blockchain source in repl
+
+#### Changed
+- Revert back the REPL example to use Electrum
+- Remove the `magic` alias for `repl`
+- Require esplora feature for repl example
+
+#### Security
+- Use dirs-next instead of dirs since the latter is unmantained
+
+## [0.1.0-beta.1] - 2020-09-08
+
+### Blockchain
+#### Added
+- Lightweight Electrum client with SSL/SOCKS5 support
+- Add a generalized "Blockchain" interface
+- Add Error::OfflineClient
+- Add the Esplora backend
+- Use async I/O in the various blockchain impls
+- Compact Filters blockchain implementation
+- Add support for Tor
+- Impl OnlineBlockchain for types wrapped in Arc
+
+### Database
+#### Added
+- Add a generalized database trait and a Sled-based implementation
+- Add an in-memory database
+
+### Descriptor
+#### Added
+- Wrap Miniscript descriptors to support xpubs
+- Policy and contribution
+- Transform a descriptor into its "public" version
+- Use `miniscript::DescriptorPublicKey`
+
+### Macros
+#### Added
+- Add a feature to enable the async interface on non-wasm32 platforms
+
+### Wallet
+#### Added
+- Wallet logic
+- Add `assume_height_reached` in PSBTSatisfier
+- Add an option to change the assumed current height
+- Specify the policy branch with a map
+- Add a few commands to handle psbts
+- Add hd_keypaths to outputs
+- Add a `TxBuilder` struct to simplify `create_tx()`'s interface
+- Abstract coin selection in a separate trait
+- Refill the address pool whenever necessary
+- Implement the wallet import/export format from FullyNoded
+- Add a type convert fee units, add `Wallet::estimate_fee()`
+- TxOrdering, shuffle/bip69 support
+- Add RBF and custom versions in TxBuilder
+- Allow limiting the use of internal utxos in TxBuilder
+- Add `force_non_witness_utxo()` to TxBuilder
+- RBF and add a few tests
+- Add AddressValidators
+- Add explicit ordering for the signers
+- Support signing the whole tx instead of individual inputs
+- Create a PSBT signer from an ExtendedDescriptor
+
+### Examples
+#### Added
+- Add REPL broadcast command
+- Add a miniscript compiler CLI
+- Expose list_transactions() in the REPL
+- Use `MemoryDatabase` in the compiler example
+- Make the REPL return JSON
+
+[unreleased]: https://github.com/bitcoindevkit/bdk/compare/v0.9.0...HEAD
+[0.1.0-beta.1]: https://github.com/bitcoindevkit/bdk/compare/96c87ea5...0.1.0-beta.1
+[v0.2.0]: https://github.com/bitcoindevkit/bdk/compare/0.1.0-beta.1...v0.2.0
+[v0.3.0]: https://github.com/bitcoindevkit/bdk/compare/v0.2.0...v0.3.0
+[v0.4.0]: https://github.com/bitcoindevkit/bdk/compare/v0.3.0...v0.4.0
+[v0.5.0]: https://github.com/bitcoindevkit/bdk/compare/v0.4.0...v0.5.0
+[v0.5.1]: https://github.com/bitcoindevkit/bdk/compare/v0.5.0...v0.5.1
+[v0.6.0]: https://github.com/bitcoindevkit/bdk/compare/v0.5.1...v0.6.0
+[v0.7.0]: https://github.com/bitcoindevkit/bdk/compare/v0.6.0...v0.7.0
+[v0.8.0]: https://github.com/bitcoindevkit/bdk/compare/v0.7.0...v0.8.0
+[v0.9.0]: https://github.com/bitcoindevkit/bdk/compare/v0.8.0...v0.9.0

CONTRIBUTING.md

@@ -0,0 +1,103 @@
+Contributing to BDK
+==============================
+
+The BDK project operates an open contributor model where anyone is welcome to
+contribute towards development in the form of peer review, documentation,
+testing and patches.
+
+Anyone is invited to contribute without regard to technical experience,
+"expertise", OSS experience, age, or other concern. However, the development of
+cryptocurrencies demands a high-level of rigor, adversarial thinking, thorough
+testing and risk-minimization.
+Any bug may cost users real money. That being said, we deeply welcome people
+contributing for the first time to an open source project or picking up Rust while
+contributing. Don't be shy, you'll learn.
+
+Communications Channels
+-----------------------
+
+Communication about BDK happens primarily on the [BDK Discord](https://discord.gg/dstn4dQ).
+
+Discussion about code base improvements happens in GitHub [issues](https://github.com/bitcoindevkit/bdk/issues) and
+on [pull requests](https://github.com/bitcoindevkit/bdk/pulls).
+
+Contribution Workflow
+---------------------
+
+The codebase is maintained using the "contributor workflow" where everyone
+without exception contributes patch proposals using "pull requests". This
+facilitates social contribution, easy testing and peer review.
+
+To contribute a patch, the worflow is a as follows:
+
+  1. Fork Repository
+  2. Create topic branch
+  3. Commit patches
+
+In general commits should be atomic and diffs should be easy to read.
+For this reason do not mix any formatting fixes or code moves with actual code
+changes. Further, each commit, individually, should compile and pass tests, in
+order to ensure git bisect and other automated tools function properly.
+
+When adding a new feature, thought must be given to the long term technical
+debt.
+Every new feature should be covered by functional tests where possible.
+
+When refactoring, structure your PR to make it easy to review and don't
+hesitate to split it into multiple small, focused PRs.
+
+The Minimal Supported Rust Version is 1.46 (enforced by our CI).
+
+Commits should cover both the issue fixed and the solution's rationale.
+These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind.
+
+To facilitate communication with other contributors, the project is making use
+of GitHub's "assignee" field. First check that no one is assigned and then
+comment suggesting that you're working on it. If someone is already assigned,
+don't hesitate to ask if the assigned party or previous commenters are still
+working on it if it has been awhile.
+
+Peer review
+-----------
+
+Anyone may participate in peer review which is expressed by comments in the
+pull request. Typically reviewers will review the code for obvious errors, as
+well as test out the patch set and opine on the technical merits of the patch.
+PR should be reviewed first on the conceptual level before focusing on code
+style or grammar fixes.
+
+Coding Conventions
+------------------
+
+This codebase uses spaces, not tabs.
+Use `cargo fmt` with the default settings to format code before committing.
+This is also enforced by the CI.
+
+Security
+--------
+
+Security is a high priority of BDK; disclosure of security vulnerabilites helps
+prevent user loss of funds.
+
+Note that BDK is currently considered "pre-production" during this time, there
+is no special handling of security issues. Please simply open an issue on
+Github.
+
+Testing
+-------
+
+Related to the security aspect, BDK developers take testing very seriously.
+Due to the modular nature of the project, writing new functional tests is easy
+and good test coverage of the codebase is an important goal.
+Refactoring the project to enable fine-grained unit testing is also an ongoing
+effort.
+
+Going further
+-------------
+
+You may be interested by Jon Atacks guide on [How to review Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-review-bitcoin-core-prs.md)
+and [How to make Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-make-bitcoin-core-prs.md).
+While there are differences between the projects in terms of context and
+maturity, many of the suggestions offered apply to this project.
+
+Overall, have fun :)

Cargo.toml

@@ -1,93 +1,94 @@
 [package]
-name = "magical"
-version = "0.1.0"
+name = "bdk"
+version = "0.9.1-dev"
 edition = "2018"
 authors = ["Alekos Filini <alekos.filini@gmail.com>", "Riccardo Casatta <riccardo@casatta.it>"]
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk"
+description = "A modern, lightweight, descriptor-based wallet library"
+keywords = ["bitcoin", "wallet", "descriptor", "psbt"]
+readme = "README.md"
+license = "MIT OR Apache-2.0"
 
 [dependencies]
-magical-macros = { version = "0.1.0-beta.1", path = "./macros" }
+bdk-macros = "^0.4"
 log = "^0.4"
-bitcoin = { version = "0.23", features = ["use-serde"] }
-miniscript = { version = "1.0" }
+miniscript = "5.1"
+bitcoin = { version = "~0.26.2", features = ["use-serde", "base64"] }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
 rand = "^0.7"
 
 # Optional dependencies
 sled = { version = "0.34", optional = true }
-electrum-client = { version = "0.2.0-beta.1", optional = true }
-reqwest = { version = "0.10", optional = true, features = ["json"] }
+electrum-client = { version = "0.7", optional = true }
+reqwest = { version = "0.11", optional = true, features = ["json"] }
 futures = { version = "0.3", optional = true }
-clap = { version = "2.33", optional = true }
-base64 = { version = "^0.11", optional = true }
 async-trait = { version = "0.1", optional = true }
 rocksdb = { version = "0.14", optional = true }
+cc = { version = ">=1.0.64", optional = true }
 socks = { version = "0.3", optional = true }
 lazy_static = { version = "1.4", optional = true }
+tiny-bip39 = { version = "^0.8", optional = true }
+bitcoinconsensus = { version = "0.19.0-3", optional = true }
 
-[patch.crates-io]
-bitcoin = { git = "https://github.com/rust-bitcoin/rust-bitcoin/", rev = "478e091" }
-miniscript = { git = "https://github.com/MagicalBitcoin/rust-miniscript", branch = "descriptor-public-key" }
+# Needed by bdk_blockchain_tests macro
+bitcoincore-rpc = { version = "0.13", optional = true }
+serial_test = { version = "0.4", optional = true }
 
 # Platform-specific dependencies
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-tokio = { version = "0.2", features = ["rt-core"] }
+tokio = { version = "1", features = ["rt"] }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 async-trait = "0.1"
 js-sys = "0.3"
+rand = { version = "^0.7", features = ["wasm-bindgen"] }
 
 [features]
 minimal = []
-compiler = ["clap", "miniscript/compiler"]
+compiler = ["miniscript/compiler"]
+verify = ["bitcoinconsensus"]
 default = ["key-value-db", "electrum"]
 electrum = ["electrum-client"]
 esplora = ["reqwest", "futures"]
-compact_filters = ["rocksdb", "socks", "lazy_static"]
+compact_filters = ["rocksdb", "socks", "lazy_static", "cc"]
 key-value-db = ["sled"]
-cli-utils = ["clap", "base64"]
 async-interface = ["async-trait"]
+all-keys = ["keys-bip39"]
+keys-bip39 = ["tiny-bip39"]
+rpc = ["bitcoincore-rpc"]
+
 
 # Debug/Test features
-debug-proc-macros = ["magical-macros/debug", "magical-testutils-macros/debug"]
+test-blockchains = ["bitcoincore-rpc", "electrum-client"]
 test-electrum = ["electrum"]
-test-md-docs = ["base64", "electrum"]
+test-rpc = ["rpc"]
+test-esplora = ["esplora"]
+test-md-docs = ["electrum"]
 
 [dev-dependencies]
-magical-testutils = { version = "0.1.0-beta.1", path = "./testutils" }
-magical-testutils-macros = { version = "0.1.0-beta.1", path = "./testutils-macros" }
-serial_test = "0.4"
 lazy_static = "1.4"
-rustyline = "6.0"
-dirs = "2.0"
 env_logger = "0.7"
-rand = "0.7"
+clap = "2.33"
+serial_test = "0.4"
+bitcoind = "0.10.0"
 
-[[example]]
-name = "repl"
-required-features = ["cli-utils"]
-[[example]]
-name = "parse_descriptor"
 [[example]]
 name = "address_validator"
+[[example]]
+name = "compact_filters_balance"
+required-features = ["compact_filters"]
 
 [[example]]
 name = "miniscriptc"
 path = "examples/compiler.rs"
 required-features = ["compiler"]
 
-# Provide a more user-friendly alias for the REPL
-[[example]]
-name = "magic"
-path = "examples/repl.rs"
-required-features = ["cli-utils"]
-
 [workspace]
-members = ["macros", "testutils", "testutils-macros"]
-
-# Generate docs with nightly to add the "features required" badge
-# https://stackoverflow.com/questions/61417452/how-to-get-a-feature-requirement-tag-in-the-documentation-generated-by-cargo-do
+members = ["macros"]
 [package.metadata.docs.rs]
-features = ["compiler", "electrum", "esplora", "compact_filters", "key-value-db"]
+features = ["compiler", "electrum", "esplora", "compact_filters", "rpc", "key-value-db", "all-keys", "verify"]
 # defines the configuration attribute `docsrs`
 rustdoc-args = ["--cfg", "docsrs"]

DEVELOPMENT_CYCLE.md

@@ -0,0 +1,46 @@
+# Development Cycle
+
+This project follows a regular releasing schedule similar to the one [used by the Rust language](https://doc.rust-lang.org/book/appendix-07-nightly-rust.html). In short, this means that a new release is made at a regular cadence, with all the feature/bugfixes that made it to `master` in time. This ensures that we don't keep delaying releases waiting for "just one more little thing".
+
+We decided to maintain a faster release cycle while the library is still in "beta", i.e. before release `1.0.0`: since we are constantly adding new features and, even more importantly, fixing issues, we want developers to have access to those updates as fast as possible. For this reason we will make a release **every 4 weeks**.
+
+Once the project will have reached a more mature state (>= `1.0.0`), we will very likely switch to longer release cycles of **6 weeks**.
+
+The "feature freeze" will happen **one week before the release date**. This means a new branch will be created originating from the `master` tip at that time, and in that branch we will stop adding new features and only focus on ensuring the ones we've added are working properly.
+
+```
+master:           - - - - * - - - * - - - - - - * - - - * ...
+                          |      /              |       |
+release/0.x.0:            * - - #               |       |
+                                                |      /
+release/0.y.0:                                  * - - #
+```
+
+As soon as the release is tagged and published, the `release` branch will be merged back into `master` to update the version in the `Cargo.toml` to apply the new `Cargo.toml` version and all the other fixes made during the feature freeze window.
+
+## Making the Release
+
+What follows are notes and procedures that maintainers can refer to when making releases. All the commits and tags must be signed and, ideally, also [timestamped](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md).
+
+Pre-`v1.0.0` our "major" releases only affect the "minor" semver value. Accordingly, our "minor" releases will only affect the "patch" value.
+
+1. Create a new branch called `release/x.y.z` from `master`. Double check that your local `master` is up-to-date with the upstream repo before doing so.
+2. Make a commit on the release branch to bump the version to `x.y.z-rc.1`. The message should be "Bump version to x.y.z-rc.1".
+3. Push the new branch to `bitcoindevkit/bdk` on GitHub.
+4. During the one week of feature freeze run additional tests on the release branch.
+5. If a bug is found:
+    - If it's a minor issue you can just fix it in the release branch, since it will be merged back to `master` eventually
+    - For bigger issues you can fix them on `master` and then *cherry-pick* the commit to the release branch
+6. Update the changelog with the new release version.
+7. Update `src/lib.rs` with the new version (line ~59)
+8. On release day, make a commit on the release branch to bump the version to `x.y.z`. The message should be "Bump version to x.y.z".
+9. Add a tag to this commit. The tag name should be `vx.y.z` (for example `v0.5.0`), and the message "Release x.y.z". Make sure the tag is signed, for extra safety use the explicit `--sign` flag.
+10. Push the new commits to the upstream release branch, wait for the CI to finish one last time.
+11. Publish **all** the updated crates to crates.io.
+12. Make a new commit to bump the version value to `x.y.(z+1)-dev`. The message should be "Bump version to x.y.(z+1)-dev".
+13. Merge the release branch back into `master`.
+14. If the `master` branch contains any unreleased changes to the `bdk-macros`, `bdk-testutils`, or `bdk-testutils-macros` crates, change the `bdk` Cargo.toml `[dev-dependencies]` to point to the local path (ie. `bdk-testutils-macros = { path = "./testutils-macros"}`)
+15. Create the release on GitHub: go to "tags", click on the dots on the right and select "Create Release". Then set the title to `vx.y.z` and write down some brief release notes.
+16. Make sure the new release shows up on crates.io and that the docs are built correctly on docs.rs.
+17. Announce the release on Twitter, Discord and Telegram.
+18. Celebrate :tada:

LICENSE

@@ -1,21 +1,14 @@
-MIT License
+This software is licensed under [Apache 2.0](LICENSE-APACHE) or
+[MIT](LICENSE-MIT), at your option.
 
-Copyright (c) 2020 Magical Bitcoin
+Some files retain their own copyright notice, however, for full authorship
+information, see version control history.
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+Except as otherwise noted in individual files, all files in this repository are
+licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+http://www.apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
+http://opensource.org/licenses/MIT>, at your option.
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+You may not use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of this software or any files in this repository except in
+accordance with one or both of these licenses.

LICENSE-APACHE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE-MIT

@@ -0,0 +1,16 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -1,29 +1,32 @@
 <div align="center">
-  <h1>Magical Bitcoin Library</h1>
+  <h1>BDK</h1>
 
-  <img src="./static/wizard.svg" width="220" />
+  <img src="./static/bdk.svg" width="220" />
 
   <p>
     <strong>A modern, lightweight, descriptor-based wallet library written in Rust!</strong>
   </p>
 
   <p>
-    <!-- <a href="https://crates.io/crates/magical"><img alt="Crate Info" src="https://img.shields.io/crates/v/magical.svg"/></a> -->
-    <a href="https://travis-ci.org/MagicalBitcoin/magical-bitcoin-wallet"><img alt="Traivs Status" src="https://travis-ci.org/MagicalBitcoin/magical-bitcoin-wallet.svg?branch=master"></a>
-    <a href="https://magicalbitcoin.org/docs-rs/magical"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-magical-green"/></a>
-    <a href="https://blog.rust-lang.org/2020/07/16/Rust-1.45.0.html"><img alt="Rustc Version 1.45+" src="https://img.shields.io/badge/rustc-1.45%2B-lightgrey.svg"/></a>
+    <a href="https://crates.io/crates/bdk"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk.svg"/></a>
+    <a href="https://github.com/bitcoindevkit/bdk/blob/master/LICENSE"><img alt="MIT or Apache-2.0 Licensed" src="https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg"/></a>
+    <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
+    <a href="https://codecov.io/gh/bitcoindevkit/bdk"><img src="https://codecov.io/gh/bitcoindevkit/bdk/branch/master/graph/badge.svg"/></a>
+    <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
+    <a href="https://blog.rust-lang.org/2020/08/27/Rust-1.46.0.html"><img alt="Rustc Version 1.46+" src="https://img.shields.io/badge/rustc-1.46%2B-lightgrey.svg"/></a>
+    <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
 
   <h4>
-    <a href="https://magicalbitcoin.org">Project Homepage</a>
+    <a href="https://bitcoindevkit.org">Project Homepage</a>
     <span> | </span>
-    <a href="https://magicalbitcoin.org/docs-rs/magical">Documentation</a>
+    <a href="https://docs.rs/bdk">Documentation</a>
   </h4>
 </div>
 
 ## About
 
-The `magical` library aims to be the core building block for Bitcoin wallets of any kind.
+The `bdk` library aims to be the core building block for Bitcoin wallets of any kind.
 
 * It uses [Miniscript](https://github.com/rust-bitcoin/rust-miniscript) to support descriptors with generalized conditions. This exact same library can be used to build
   single-sig wallets, multisigs, timelocked contracts and more.
@@ -35,15 +38,15 @@ The `magical` library aims to be the core building block for Bitcoin wallets of
 
 ### Sync the balance of a descriptor
 
-```no_run
-use magical::Wallet;
-use magical::database::MemoryDatabase;
-use magical::blockchain::{noop_progress, ElectrumBlockchain};
+```rust,no_run
+use bdk::Wallet;
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
 
-use magical::electrum_client::Client;
+use bdk::electrum_client::Client;
 
-fn main() -> Result<(), magical::Error> {
-    let client = Client::new("ssl://electrum.blockstream.info:60002", None)?;
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
     let wallet = Wallet::new(
         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
@@ -62,21 +65,21 @@ fn main() -> Result<(), magical::Error> {
 
 ### Generate a few addresses
 
-```
-use magical::{Wallet, OfflineWallet};
-use magical::database::MemoryDatabase;
+```rust
+use bdk::{Wallet, database::MemoryDatabase};
+use bdk::wallet::AddressIndex::New;
 
-fn main() -> Result<(), magical::Error> {
-    let wallet: OfflineWallet<_> = Wallet::new_offline(
+fn main() -> Result<(), bdk::Error> {
+    let wallet = Wallet::new_offline(
         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
         bitcoin::Network::Testnet,
         MemoryDatabase::default(),
     )?;
 
-    println!("Address #0: {}", wallet.get_new_address()?);
-    println!("Address #1: {}", wallet.get_new_address()?);
-    println!("Address #2: {}", wallet.get_new_address()?);
+    println!("Address #0: {}", wallet.get_address(New)?);
+    println!("Address #1: {}", wallet.get_address(New)?);
+    println!("Address #2: {}", wallet.get_address(New)?);
 
     Ok(())
 }
@@ -84,17 +87,18 @@ fn main() -> Result<(), magical::Error> {
 
 ### Create a transaction
 
-```no_run
-use magical::{FeeRate, TxBuilder, Wallet};
-use magical::database::MemoryDatabase;
-use magical::blockchain::{noop_progress, ElectrumBlockchain};
+```rust,no_run
+use bdk::{FeeRate, Wallet};
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
 
-use magical::electrum_client::Client;
+use bdk::electrum_client::Client;
+use bdk::wallet::AddressIndex::New;
 
 use bitcoin::consensus::serialize;
 
-fn main() -> Result<(), magical::Error> {
-    let client = Client::new("ssl://electrum.blockstream.info:60002", None)?;
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
     let wallet = Wallet::new(
         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
@@ -105,13 +109,16 @@ fn main() -> Result<(), magical::Error> {
 
     wallet.sync(noop_progress(), None)?;
 
-    let send_to = wallet.get_new_address()?;
-    let (psbt, details) = wallet.create_tx(
-        TxBuilder::with_recipients(vec![(send_to.script_pubkey(), 50_000)])
+    let send_to = wallet.get_address(New)?;
+    let (psbt, details) = {
+        let mut builder = wallet.build_tx();
+        builder
+            .add_recipient(send_to.script_pubkey(), 50_000)
             .enable_rbf()
             .do_not_spend_change()
-            .fee_rate(FeeRate::from_sat_per_vb(5.0))
-    )?;
+            .fee_rate(FeeRate::from_sat_per_vb(5.0));
+        builder.finish()?
+    };
 
     println!("Transaction details: {:#?}", details);
     println!("Unsigned PSBT: {}", base64::encode(&serialize(&psbt)));
@@ -122,14 +129,13 @@ fn main() -> Result<(), magical::Error> {
 
 ### Sign a transaction
 
-```no_run
-use magical::{Wallet, OfflineWallet};
-use magical::database::MemoryDatabase;
+```rust,no_run
+use bdk::{Wallet, SignOptions, database::MemoryDatabase};
 
 use bitcoin::consensus::deserialize;
 
-fn main() -> Result<(), magical::Error> {
-    let wallet: OfflineWallet<_> = Wallet::new_offline(
+fn main() -> Result<(), bdk::Error> {
+    let wallet = Wallet::new_offline(
         "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"),
         bitcoin::Network::Testnet,
@@ -137,10 +143,27 @@ fn main() -> Result<(), magical::Error> {
     )?;
 
     let psbt = "...";
-    let psbt = deserialize(&base64::decode(psbt).unwrap())?;
+    let mut psbt = deserialize(&base64::decode(psbt).unwrap())?;
 
-    let (signed_psbt, finalized) = wallet.sign(psbt, None)?;
+    let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
 
     Ok(())
 }
 ```
+
+## License
+
+Licensed under either of
+
+ * Apache License, Version 2.0
+   ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+ * MIT license
+   ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+
+at your option.
+
+## Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in the work by you, as defined in the Apache-2.0 license, shall be
+dual licensed as above, without any additional terms or conditions.

ci/start-core.sh

@@ -1,24 +1,14 @@
 #!/usr/bin/env sh
 
-set -e
+echo "Starting bitcoin node."
+mkdir $GITHUB_WORKSPACE/.bitcoin
+/root/bitcoind -regtest -server -daemon -datadir=$GITHUB_WORKSPACE/.bitcoin -fallbackfee=0.0002 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -blockfilterindex=1 -peerblockfilters=1
 
-BITCOIN_VERSION=0.20.1
-
-# This should be cached by Travis
-cargo install --git https://github.com/romanz/electrs --bin electrs
-
-curl -O -L https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz
-tar xf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz
-
-export PATH=$PATH:./bitcoin-$BITCOIN_VERSION/bin
-
-bitcoind -regtest=1 -daemon=1 -fallbackfee=0.0002
-until bitcoin-cli -regtest getblockchaininfo; do
+echo "Waiting for bitcoin node."
+until /root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin getblockchaininfo; do
     sleep 1
 done
-
-ADDR=$(bitcoin-cli -regtest getnewaddress)
-bitcoin-cli -regtest generatetoaddress 150 $ADDR
-
-nohup electrs --network regtest --jsonrpc-import --cookie-file /home/travis/.bitcoin/regtest/.cookie &
-sleep 5
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin createwallet $BDK_RPC_WALLET
+echo "Generating 150 bitcoin blocks."
+ADDR=$(/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin -rpcwallet=$BDK_RPC_WALLET getnewaddress)
+/root/bitcoin-cli -regtest -datadir=$GITHUB_WORKSPACE/.bitcoin generatetoaddress 150 $ADDR

codecov.yaml

@@ -0,0 +1,13 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+        base: auto
+        informational: false
+    patch:
+      default:
+        target: auto
+        threshold: 100%
+        base: auto

examples/address_validator.rs

@@ -1,46 +1,35 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::sync::Arc;
 
-use magical::bitcoin;
-use magical::database::MemoryDatabase;
-use magical::descriptor::HDKeyPaths;
-use magical::wallet::address_validator::{AddressValidator, AddressValidatorError};
-use magical::ScriptType;
-use magical::{OfflineWallet, Wallet};
+use bdk::bitcoin;
+use bdk::database::MemoryDatabase;
+use bdk::descriptor::HdKeyPaths;
+use bdk::wallet::address_validator::{AddressValidator, AddressValidatorError};
+use bdk::KeychainKind;
+use bdk::Wallet;
 
+use bdk::wallet::AddressIndex::New;
 use bitcoin::hashes::hex::FromHex;
 use bitcoin::util::bip32::Fingerprint;
 use bitcoin::{Network, Script};
 
+#[derive(Debug)]
 struct DummyValidator;
 impl AddressValidator for DummyValidator {
     fn validate(
         &self,
-        script_type: ScriptType,
-        hd_keypaths: &HDKeyPaths,
+        keychain: KeychainKind,
+        hd_keypaths: &HdKeyPaths,
         script: &Script,
     ) -> Result<(), AddressValidatorError> {
         let (_, path) = hd_keypaths
@@ -50,23 +39,23 @@ impl AddressValidator for DummyValidator {
 
         println!(
             "Validating `{:?}` {} address, script: {}",
-            script_type, path, script
+            keychain, path, script
         );
 
         Ok(())
     }
 }
 
-fn main() -> Result<(), magical::Error> {
+fn main() -> Result<(), bdk::Error> {
     let descriptor = "sh(and_v(v:pk(tpubDDpWvmUrPZrhSPmUzCMBHffvC3HyMAPnWDSAQNBTnj1iZeJa7BZQEttFiP4DS4GCcXQHezdXhn86Hj6LHX5EDstXPWrMaSneRWM8yUf6NFd/*),after(630000)))";
-    let mut wallet: OfflineWallet<_> =
+    let mut wallet =
         Wallet::new_offline(descriptor, None, Network::Regtest, MemoryDatabase::new())?;
 
-    wallet.add_address_validator(Arc::new(Box::new(DummyValidator)));
+    wallet.add_address_validator(Arc::new(DummyValidator));
 
-    wallet.get_new_address()?;
-    wallet.get_new_address()?;
-    wallet.get_new_address()?;
+    wallet.get_address(New)?;
+    wallet.get_address(New)?;
+    wallet.get_address(New)?;
 
     Ok(())
 }

examples/compact_filters_balance.rs

@@ -0,0 +1,43 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+use bdk::blockchain::compact_filters::*;
+use bdk::blockchain::noop_progress;
+use bdk::database::MemoryDatabase;
+use bdk::*;
+use bitcoin::*;
+use blockchain::compact_filters::CompactFiltersBlockchain;
+use blockchain::compact_filters::CompactFiltersError;
+use log::info;
+use std::sync::Arc;
+
+/// This will return wallet balance using compact filters
+/// Requires a synced local bitcoin node 0.21 running on testnet with blockfilterindex=1 and peerblockfilters=1
+fn main() -> Result<(), CompactFiltersError> {
+    env_logger::init();
+    info!("start");
+
+    let num_threads = 4;
+    let mempool = Arc::new(Mempool::default());
+    let peers = (0..num_threads)
+        .map(|_| Peer::connect("localhost:18333", Arc::clone(&mempool), Network::Testnet))
+        .collect::<Result<_, _>>()?;
+    let blockchain = CompactFiltersBlockchain::new(peers, "./wallet-filters", Some(500_000))?;
+    info!("done {:?}", blockchain);
+    let descriptor = "wpkh(tpubD6NzVbkrYhZ4X2yy78HWrr1M9NT8dKeWfzNiQqDdMqqa9UmmGztGGz6TaLFGsLfdft5iu32gxq1T4eMNxExNNWzVCpf9Y6JZi5TnqoC9wJq/*)";
+
+    let database = MemoryDatabase::default();
+    let wallet =
+        Arc::new(Wallet::new(descriptor, None, Network::Testnet, database, blockchain).unwrap());
+    wallet.sync(noop_progress(), None).unwrap();
+    info!("balance: {}", wallet.get_balance()?);
+    Ok(())
+}

examples/compiler.rs

@@ -1,34 +1,22 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
+extern crate bdk;
 extern crate bitcoin;
 extern crate clap;
 extern crate log;
-extern crate magical;
 extern crate miniscript;
 extern crate serde_json;
 
+use std::error::Error;
 use std::str::FromStr;
 
 use log::info;
@@ -39,10 +27,11 @@ use bitcoin::Network;
 use miniscript::policy::Concrete;
 use miniscript::Descriptor;
 
-use magical::database::memory::MemoryDatabase;
-use magical::{OfflineWallet, ScriptType, Wallet};
+use bdk::database::memory::MemoryDatabase;
+use bdk::wallet::AddressIndex::New;
+use bdk::{KeychainKind, Wallet};
 
-fn main() {
+fn main() -> Result<(), Box<dyn Error>> {
     env_logger::init_from_env(
         env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
     );
@@ -74,19 +63,19 @@ fn main() {
                 .help("Sets the network")
                 .takes_value(true)
                 .default_value("testnet")
-                .possible_values(&["testnet", "regtest"]),
+                .possible_values(&["testnet", "regtest", "bitcoin", "signet"]),
         )
         .get_matches();
 
     let policy_str = matches.value_of("POLICY").unwrap();
     info!("Compiling policy: {}", policy_str);
 
-    let policy = Concrete::<String>::from_str(&policy_str).unwrap();
+    let policy = Concrete::<String>::from_str(&policy_str)?;
 
     let descriptor = match matches.value_of("TYPE").unwrap() {
-        "sh" => Descriptor::Sh(policy.compile().unwrap()),
-        "wsh" => Descriptor::Wsh(policy.compile().unwrap()),
-        "sh-wsh" => Descriptor::ShWsh(policy.compile().unwrap()),
+        "sh" => Descriptor::new_sh(policy.compile()?)?,
+        "wsh" => Descriptor::new_wsh(policy.compile()?)?,
+        "sh-wsh" => Descriptor::new_sh_wsh(policy.compile()?)?,
         _ => panic!("Invalid type"),
     };
 
@@ -94,20 +83,23 @@ fn main() {
 
     let database = MemoryDatabase::new();
 
-    let network = match matches.value_of("network") {
-        Some("regtest") => Network::Regtest,
-        Some("testnet") | _ => Network::Testnet,
-    };
-    let wallet: OfflineWallet<_> =
-        Wallet::new_offline(&format!("{}", descriptor), None, network, database).unwrap();
+    let network = matches
+        .value_of("network")
+        .map(|n| Network::from_str(n))
+        .transpose()
+        .unwrap()
+        .unwrap_or(Network::Testnet);
+    let wallet = Wallet::new_offline(&format!("{}", descriptor), None, network, database)?;
 
-    info!("... First address: {}", wallet.get_new_address().unwrap());
+    info!("... First address: {}", wallet.get_address(New)?);
 
     if matches.is_present("parsed_policy") {
-        let spending_policy = wallet.policies(ScriptType::External).unwrap();
+        let spending_policy = wallet.policies(KeychainKind::External)?;
         info!(
             "... Spending policy:\n{}",
-            serde_json::to_string_pretty(&spending_policy).unwrap()
+            serde_json::to_string_pretty(&spending_policy)?
         );
     }
+
+    Ok(())
 }

examples/parse_descriptor.rs

@@ -1,57 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-extern crate magical;
-extern crate serde_json;
-
-use std::sync::Arc;
-
-use magical::bitcoin::util::bip32::ChildNumber;
-use magical::bitcoin::*;
-use magical::descriptor::*;
-
-fn main() {
-    let desc = "wsh(or_d(\
-                    multi(\
-                      2,[d34db33f/44'/0'/0']xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*,tprv8ZgxMBicQKsPduL5QnGihpprdHyypMGi4DhimjtzYemu7se5YQNcZfAPLqXRuGHb5ZX2eTQj62oNqMnyxJ7B7wz54Uzswqw8fFqMVdcmVF7/1/*\
-                    ),\
-                    and_v(vc:pk_h(cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy),older(1000))\
-                   ))";
-
-    let (extended_desc, key_map) = ExtendedDescriptor::parse_secret(desc).unwrap();
-    println!("{:?}", extended_desc);
-
-    let signers = Arc::new(key_map.into());
-    let policy = extended_desc.extract_policy(signers).unwrap();
-    println!("policy: {}", serde_json::to_string(&policy).unwrap());
-
-    let derived_desc = extended_desc.derive(&[ChildNumber::from_normal_idx(42).unwrap()]);
-    println!("{:?}", derived_desc);
-
-    let addr = derived_desc.address(Network::Testnet).unwrap();
-    println!("{}", addr);
-
-    let script = derived_desc.witness_script();
-    println!("{:?}", script);
-}

examples/repl.rs

@@ -1,144 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-use std::fs;
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use rustyline::error::ReadlineError;
-use rustyline::Editor;
-
-use clap::AppSettings;
-
-#[allow(unused_imports)]
-use log::{debug, error, info, trace, LevelFilter};
-
-use bitcoin::Network;
-
-use magical::bitcoin;
-use magical::blockchain::compact_filters::*;
-use magical::cli;
-use magical::sled;
-use magical::Wallet;
-
-fn prepare_home_dir() -> PathBuf {
-    let mut dir = PathBuf::new();
-    dir.push(&dirs::home_dir().unwrap());
-    dir.push(".magical-bitcoin");
-
-    if !dir.exists() {
-        info!("Creating home directory {}", dir.as_path().display());
-        fs::create_dir(&dir).unwrap();
-    }
-
-    dir.push("database.sled");
-    dir
-}
-
-fn main() {
-    env_logger::init();
-
-    let app = cli::make_cli_subcommands();
-    let mut repl_app = app.clone().setting(AppSettings::NoBinaryName);
-
-    let app = cli::add_global_flags(app);
-
-    let matches = app.get_matches();
-
-    // TODO
-    // let level = match matches.occurrences_of("v") {
-    //     0 => LevelFilter::Info,
-    //     1 => LevelFilter::Debug,
-    //     _ => LevelFilter::Trace,
-    // };
-
-    let network = match matches.value_of("network") {
-        Some("regtest") => Network::Regtest,
-        Some("testnet") | _ => Network::Testnet,
-    };
-
-    let descriptor = matches.value_of("descriptor").unwrap();
-    let change_descriptor = matches.value_of("change_descriptor");
-    debug!("descriptors: {:?} {:?}", descriptor, change_descriptor);
-
-    let database = sled::open(prepare_home_dir().to_str().unwrap()).unwrap();
-    let tree = database
-        .open_tree(matches.value_of("wallet").unwrap())
-        .unwrap();
-    debug!("database opened successfully");
-
-    let num_threads = 1;
-
-    let mempool = Arc::new(Mempool::default());
-    let peers = (0..num_threads)
-        .map(|_| Peer::connect("192.168.1.136:8333", Arc::clone(&mempool), Network::Bitcoin))
-        .collect::<Result<_, _>>()
-        .unwrap();
-    let blockchain =
-        CompactFiltersBlockchain::new(peers, "./wallet-filters", Some(500_000)).unwrap();
-
-    let wallet = Wallet::new(descriptor, change_descriptor, network, tree, blockchain).unwrap();
-    let wallet = Arc::new(wallet);
-
-    if let Some(_sub_matches) = matches.subcommand_matches("repl") {
-        let mut rl = Editor::<()>::new();
-
-        // if rl.load_history("history.txt").is_err() {
-        //     println!("No previous history.");
-        // }
-
-        loop {
-            let readline = rl.readline(">> ");
-            match readline {
-                Ok(line) => {
-                    if line.trim() == "" {
-                        continue;
-                    }
-
-                    rl.add_history_entry(line.as_str());
-                    let matches = repl_app.get_matches_from_safe_borrow(line.split(" "));
-                    if let Err(err) = matches {
-                        println!("{}", err.message);
-                        continue;
-                    }
-
-                    let result =
-                        cli::handle_matches(&Arc::clone(&wallet), matches.unwrap()).unwrap();
-                    println!("{}", serde_json::to_string_pretty(&result).unwrap());
-                }
-                Err(ReadlineError::Interrupted) => continue,
-                Err(ReadlineError::Eof) => break,
-                Err(err) => {
-                    println!("{:?}", err);
-                    break;
-                }
-            }
-        }
-
-    // rl.save_history("history.txt").unwrap();
-    } else {
-        let result = cli::handle_matches(&wallet, matches).unwrap();
-        println!("{}", serde_json::to_string_pretty(&result).unwrap());
-    }
-}

macros/Cargo.toml

@@ -1,8 +1,14 @@
 [package]
-name = "magical-macros"
-version = "0.1.0-beta.1"
+name = "bdk-macros"
+version = "0.4.0"
 authors = ["Alekos Filini <alekos.filini@gmail.com>"]
 edition = "2018"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk-macros"
+description = "Supporting macros for `bdk`"
+keywords = ["bdk"]
+license = "MIT OR Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

macros/src/lib.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 #[macro_use]
 extern crate quote;
@@ -145,7 +132,7 @@ pub fn await_or_block(expr: TokenStream) -> TokenStream {
         {
             #[cfg(all(not(target_arch = "wasm32"), not(feature = "async-interface")))]
             {
-                tokio::runtime::Runtime::new().unwrap().block_on(#expr)
+                tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(#expr)
             }
 
             #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]

run_blockchain_tests.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+
+usage() {
+    cat <<'EOF'
+Script for running the bdk blockchain tests for a specific blockchain by starting up the backend in docker.
+
+Usage: ./run_blockchain_tests.sh [esplora|electrum|rpc] [test name].
+
+EOF
+}
+
+eprintln(){
+    echo "$@" >&2
+}
+
+cleanup() {
+    if test "$id"; then
+        eprintln "cleaning up $blockchain docker container $id";
+        docker rm -fv "$id" > /dev/null;
+        rm /tmp/regtest-"$id".cookie;
+    fi
+    trap - EXIT INT
+}
+
+# Makes sure we clean up the container at the end or if ^C
+trap 'rc=$?; cleanup; exit $rc' EXIT INT
+
+blockchain="$1"
+test_name="$2"
+
+case "$blockchain" in
+    electrum)
+        eprintln "starting electrs docker container"
+        id="$(docker run --detach -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp bitcoindevkit/electrs:0.4.0)"
+        ;;
+    esplora)
+        eprintln "starting esplora docker container"
+        id="$(docker run --detach -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp -p 127.0.0.1:3002:3002/tcp bitcoindevkit/esplora:0.4.0)"
+        export BDK_ESPLORA_URL=http://127.0.0.1:3002
+        ;;
+    rpc)
+        eprintln "starting bitcoind docker container (via electrs container)"
+        id="$(docker run --detach -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp bitcoindevkit/electrs:0.4.0)"      
+        ;;
+    *)
+        usage;
+        exit 1;
+        ;;
+    esac
+
+# taken from https://github.com/bitcoindevkit/bitcoin-regtest-box
+export BDK_RPC_AUTH=COOKIEFILE
+export BDK_RPC_COOKIEFILE=/tmp/regtest-"$id".cookie
+export BDK_RPC_URL=127.0.0.1:18443
+export BDK_RPC_WALLET=bdk-test
+export BDK_ELECTRUM_URL=tcp://127.0.0.1:60401
+
+cli(){
+    docker exec -it "$id" /root/bitcoin-cli -regtest -datadir=/root/.bitcoin $@
+}
+
+#eprintln "running getwalletinfo until bitcoind seems to be alive"
+while ! cli getwalletinfo >/dev/null; do sleep 1; done
+
+# sleep again for good measure!
+sleep 1;
+
+# copy bitcoind cookie file to /tmp
+docker cp "$id":/root/.bitcoin/regtest/.cookie /tmp/regtest-"$id".cookie
+
+cargo test --features "test-blockchains,test-$blockchain" --no-default-features "$blockchain::bdk_blockchain_tests::$test_name"

src/blockchain/any.rs

@@ -0,0 +1,246 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Runtime-checked blockchain types
+//!
+//! This module provides the implementation of [`AnyBlockchain`] which allows switching the
+//! inner [`Blockchain`] type at runtime.
+//!
+//! ## Example
+//!
+//! In this example both `wallet_electrum` and `wallet_esplora` have the same type of
+//! `Wallet<AnyBlockchain, MemoryDatabase>`. This means that they could both, for instance, be
+//! assigned to a struct member.
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::blockchain::*;
+//! # use bdk::database::MemoryDatabase;
+//! # use bdk::Wallet;
+//! # #[cfg(feature = "electrum")]
+//! # {
+//! let electrum_blockchain = ElectrumBlockchain::from(electrum_client::Client::new("...")?);
+//! let wallet_electrum: Wallet<AnyBlockchain, _> = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     electrum_blockchain.into(),
+//! )?;
+//! # }
+//!
+//! # #[cfg(feature = "esplora")]
+//! # {
+//! let esplora_blockchain = EsploraBlockchain::new("...", None);
+//! let wallet_esplora: Wallet<AnyBlockchain, _> = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     esplora_blockchain.into(),
+//! )?;
+//! # }
+//!
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! When paired with the use of [`ConfigurableBlockchain`], it allows creating wallets with any
+//! blockchain type supported using a single line of code:
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::blockchain::*;
+//! # use bdk::database::MemoryDatabase;
+//! # use bdk::Wallet;
+//! let config = serde_json::from_str("...")?;
+//! let blockchain = AnyBlockchain::from_config(&config)?;
+//! let wallet = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     blockchain,
+//! )?;
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use super::*;
+
+macro_rules! impl_from {
+    ( $from:ty, $to:ty, $variant:ident, $( $cfg:tt )* ) => {
+        $( $cfg )*
+        impl From<$from> for $to {
+            fn from(inner: $from) -> Self {
+                <$to>::$variant(inner)
+            }
+        }
+    };
+}
+
+macro_rules! impl_inner_method {
+    ( $self:expr, $name:ident $(, $args:expr)* ) => {
+        match $self {
+            #[cfg(feature = "electrum")]
+            AnyBlockchain::Electrum(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "esplora")]
+            AnyBlockchain::Esplora(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "compact_filters")]
+            AnyBlockchain::CompactFilters(inner) => inner.$name( $($args, )* ),
+        }
+    }
+}
+
+/// Type that can contain any of the [`Blockchain`] types defined by the library
+///
+/// It allows switching backend at runtime
+///
+/// See [this module](crate::blockchain::any)'s documentation for a usage example.
+pub enum AnyBlockchain {
+    #[cfg(feature = "electrum")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
+    /// Electrum client
+    Electrum(electrum::ElectrumBlockchain),
+    #[cfg(feature = "esplora")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
+    /// Esplora client
+    Esplora(esplora::EsploraBlockchain),
+    #[cfg(feature = "compact_filters")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
+    /// Compact filters client
+    CompactFilters(compact_filters::CompactFiltersBlockchain),
+}
+
+#[maybe_async]
+impl Blockchain for AnyBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        maybe_await!(impl_inner_method!(self, get_capabilities))
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(
+            self,
+            setup,
+            stop_gap,
+            database,
+            progress_update
+        ))
+    }
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(
+            self,
+            sync,
+            stop_gap,
+            database,
+            progress_update
+        ))
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        maybe_await!(impl_inner_method!(self, get_tx, txid))
+    }
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(self, broadcast, tx))
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        maybe_await!(impl_inner_method!(self, get_height))
+    }
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        maybe_await!(impl_inner_method!(self, estimate_fee, target))
+    }
+}
+
+impl_from!(electrum::ElectrumBlockchain, AnyBlockchain, Electrum, #[cfg(feature = "electrum")]);
+impl_from!(esplora::EsploraBlockchain, AnyBlockchain, Esplora, #[cfg(feature = "esplora")]);
+impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilters, #[cfg(feature = "compact_filters")]);
+
+/// Type that can contain any of the blockchain configurations defined by the library
+///
+/// This allows storing a single configuration that can be loaded into an [`AnyBlockchain`]
+/// instance. Wallets that plan to offer users the ability to switch blockchain backend at runtime
+/// will find this particularly useful.
+///
+/// This type can be serialized from a JSON object like:
+///
+/// ```
+/// # #[cfg(feature = "electrum")]
+/// # {
+/// use bdk::blockchain::{electrum::ElectrumBlockchainConfig, AnyBlockchainConfig};
+/// let config: AnyBlockchainConfig = serde_json::from_str(
+///     r#"{
+///    "type" : "electrum",
+///    "url" : "ssl://electrum.blockstream.info:50002",
+///    "retry": 2
+/// }"#,
+/// )
+/// .unwrap();
+/// assert_eq!(
+///     config,
+///     AnyBlockchainConfig::Electrum(ElectrumBlockchainConfig {
+///         url: "ssl://electrum.blockstream.info:50002".into(),
+///         retry: 2,
+///         socks5: None,
+///         timeout: None
+///     })
+/// );
+/// # }
+/// ```
+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, PartialEq)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum AnyBlockchainConfig {
+    #[cfg(feature = "electrum")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
+    /// Electrum client
+    Electrum(electrum::ElectrumBlockchainConfig),
+    #[cfg(feature = "esplora")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
+    /// Esplora client
+    Esplora(esplora::EsploraBlockchainConfig),
+    #[cfg(feature = "compact_filters")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
+    /// Compact filters client
+    CompactFilters(compact_filters::CompactFiltersBlockchainConfig),
+}
+
+impl ConfigurableBlockchain for AnyBlockchain {
+    type Config = AnyBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(match config {
+            #[cfg(feature = "electrum")]
+            AnyBlockchainConfig::Electrum(inner) => {
+                AnyBlockchain::Electrum(electrum::ElectrumBlockchain::from_config(inner)?)
+            }
+            #[cfg(feature = "esplora")]
+            AnyBlockchainConfig::Esplora(inner) => {
+                AnyBlockchain::Esplora(esplora::EsploraBlockchain::from_config(inner)?)
+            }
+            #[cfg(feature = "compact_filters")]
+            AnyBlockchainConfig::CompactFilters(inner) => AnyBlockchain::CompactFilters(
+                compact_filters::CompactFiltersBlockchain::from_config(inner)?,
+            ),
+        })
+    }
+}
+
+impl_from!(electrum::ElectrumBlockchainConfig, AnyBlockchainConfig, Electrum, #[cfg(feature = "electrum")]);
+impl_from!(esplora::EsploraBlockchainConfig, AnyBlockchainConfig, Esplora, #[cfg(feature = "esplora")]);
+impl_from!(compact_filters::CompactFiltersBlockchainConfig, AnyBlockchainConfig, CompactFilters, #[cfg(feature = "compact_filters")]);

src/blockchain/compact_filters/mod.rs

@@ -1,30 +1,17 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Compact Filters
 //!
-//! This module contains a multithreaded implementation of an [`OnlineBlockchain`] backend that
+//! This module contains a multithreaded implementation of an [`Blockchain`] backend that
 //! uses BIP157 (aka "Neutrino") to populate the wallet's [database](crate::database::Database)
 //! by downloading compact filters from the P2P network.
 //!
@@ -37,25 +24,29 @@
 //! connecting to a single peer at a time, optionally by opening multiple connections if it's
 //! desirable to use multiple threads at once to sync in parallel.
 //!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
 //! ## Example
 //!
 //! ```no_run
 //! # use std::sync::Arc;
 //! # use bitcoin::*;
-//! # use magical::*;
-//! # use magical::blockchain::compact_filters::*;
+//! # use bdk::*;
+//! # use bdk::blockchain::compact_filters::*;
 //! let num_threads = 4;
 //!
 //! let mempool = Arc::new(Mempool::default());
 //! let peers = (0..num_threads)
-//!     .map(|_| Peer::connect(
-//!         "btcd-mainnet.lightning.computer:8333",
-//!         Arc::clone(&mempool),
-//!         Network::Bitcoin,
-//!     ))
+//!     .map(|_| {
+//!         Peer::connect(
+//!             "btcd-mainnet.lightning.computer:8333",
+//!             Arc::clone(&mempool),
+//!             Network::Bitcoin,
+//!         )
+//!     })
 //!     .collect::<Result<_, _>>()?;
 //! let blockchain = CompactFiltersBlockchain::new(peers, "./wallet-filters", Some(500_000))?;
-//! # Ok::<(), magical::error::Error>(())
+//! # Ok::<(), CompactFiltersError>(())
 //! ```
 
 use std::collections::HashSet;
@@ -68,7 +59,7 @@ use std::sync::{Arc, Mutex};
 use log::{debug, error, info, trace};
 
 use bitcoin::network::message_blockdata::Inventory;
-use bitcoin::{OutPoint, Transaction, Txid};
+use bitcoin::{Network, OutPoint, Transaction, Txid};
 
 use rocksdb::{Options, SliceTransform, DB};
 
@@ -76,11 +67,11 @@ mod peer;
 mod store;
 mod sync;
 
-use super::{Blockchain, Capability, OnlineBlockchain, Progress};
+use super::{Blockchain, Capability, ConfigurableBlockchain, Progress};
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{ScriptType, TransactionDetails, UTXO};
-use crate::FeeRate;
+use crate::types::{KeychainKind, LocalUtxo, TransactionDetails};
+use crate::{ConfirmationTime, FeeRate};
 
 use peer::*;
 use store::*;
@@ -97,7 +88,11 @@ const PROCESS_BLOCKS_COST: f32 = 20_000.0;
 /// ## Example
 /// See the [`blockchain::compact_filters`](crate::blockchain::compact_filters) module for a usage example.
 #[derive(Debug)]
-pub struct CompactFiltersBlockchain(Option<CompactFilters>);
+pub struct CompactFiltersBlockchain {
+    peers: Vec<Arc<Peer>>,
+    headers: Arc<ChainStore<Full>>,
+    skip_blocks: Option<usize>,
+}
 
 impl CompactFiltersBlockchain {
     /// Construct a new instance given a list of peers, a path to store headers and block
@@ -108,29 +103,6 @@ impl CompactFiltersBlockchain {
     /// in parallel. It's currently recommended to only connect to a single peer to avoid
     /// inconsistencies in the data returned, optionally with multiple connections in parallel to
     /// speed-up the sync process.
-    pub fn new<P: AsRef<Path>>(
-        peers: Vec<Peer>,
-        storage_dir: P,
-        skip_blocks: Option<usize>,
-    ) -> Result<Self, CompactFiltersError> {
-        Ok(CompactFiltersBlockchain(Some(CompactFilters::new(
-            peers,
-            storage_dir,
-            skip_blocks,
-        )?)))
-    }
-}
-
-/// Internal struct that contains the state of a [`CompactFiltersBlockchain`]
-#[derive(Debug)]
-struct CompactFilters {
-    peers: Vec<Arc<Peer>>,
-    headers: Arc<ChainStore<Full>>,
-    skip_blocks: Option<usize>,
-}
-
-impl CompactFilters {
-    /// Constructor, see [`CompactFiltersBlockchain::new`] for the documentation
     pub fn new<P: AsRef<Path>>(
         peers: Vec<Peer>,
         storage_dir: P,
@@ -146,7 +118,7 @@ impl CompactFilters {
 
         let network = peers[0].get_network();
 
-        let cfs = DB::list_cf(&opts, &storage_dir).unwrap_or(vec!["default".to_string()]);
+        let cfs = DB::list_cf(&opts, &storage_dir).unwrap_or_else(|_| vec!["default".to_string()]);
         let db = DB::open_cf(&opts, &storage_dir, &cfs)?;
         let headers = Arc::new(ChainStore::new(db, network)?);
 
@@ -160,7 +132,7 @@ impl CompactFilters {
             headers.recover_snapshot(cf_name)?;
         }
 
-        Ok(CompactFilters {
+        Ok(CompactFiltersBlockchain {
             peers: peers.into_iter().map(Arc::new).collect(),
             headers,
             skip_blocks,
@@ -174,7 +146,7 @@ impl CompactFilters {
         database: &mut D,
         tx: &Transaction,
         height: Option<u32>,
-        timestamp: u64,
+        timestamp: Option<u64>,
         internal_max_deriv: &mut Option<u32>,
         external_max_deriv: &mut Option<u32>,
     ) -> Result<(), Error> {
@@ -205,22 +177,22 @@ impl CompactFilters {
             outputs_sum += output.value;
 
             // this output is ours, we have a path to derive it
-            if let Some((script_type, child)) =
+            if let Some((keychain, child)) =
                 database.get_path_from_script_pubkey(&output.script_pubkey)?
             {
                 debug!("{} output #{} is mine, adding utxo", tx.txid(), i);
-                updates.set_utxo(&UTXO {
+                updates.set_utxo(&LocalUtxo {
                     outpoint: OutPoint::new(tx.txid(), i as u32),
                     txout: output.clone(),
-                    is_internal: script_type.is_internal(),
+                    keychain,
                 })?;
                 incoming += output.value;
 
-                if script_type == ScriptType::Internal
+                if keychain == KeychainKind::Internal
                     && (internal_max_deriv.is_none() || child > internal_max_deriv.unwrap_or(0))
                 {
                     *internal_max_deriv = Some(child);
-                } else if script_type == ScriptType::External
+                } else if keychain == KeychainKind::External
                     && (external_max_deriv.is_none() || child > external_max_deriv.unwrap_or(0))
                 {
                     *external_max_deriv = Some(child);
@@ -234,9 +206,9 @@ impl CompactFilters {
                 transaction: Some(tx.clone()),
                 received: incoming,
                 sent: outgoing,
-                height,
-                timestamp,
-                fees: inputs_sum.checked_sub(outputs_sum).unwrap_or(0),
+                confirmation_time: ConfirmationTime::new(height, timestamp),
+                verified: height.is_some(),
+                fee: Some(inputs_sum.saturating_sub(outputs_sum)),
             };
 
             info!("Saving tx {}", tx.txid);
@@ -250,46 +222,33 @@ impl CompactFilters {
 }
 
 impl Blockchain for CompactFiltersBlockchain {
-    fn offline() -> Self {
-        CompactFiltersBlockchain(None)
-    }
-
-    fn is_online(&self) -> bool {
-        self.0.is_some()
-    }
-}
-
-impl OnlineBlockchain for CompactFiltersBlockchain {
     fn get_capabilities(&self) -> HashSet<Capability> {
         vec![Capability::FullHistory].into_iter().collect()
     }
 
+    #[allow(clippy::mutex_atomic)] // Mutex is easier to understand than a CAS loop.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
         _stop_gap: Option<usize>, // TODO: move to electrum and esplora only
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-        let first_peer = &inner.peers[0];
+        let first_peer = &self.peers[0];
 
-        let skip_blocks = inner.skip_blocks.unwrap_or(0);
+        let skip_blocks = self.skip_blocks.unwrap_or(0);
 
-        let cf_sync = Arc::new(CFSync::new(Arc::clone(&inner.headers), skip_blocks, 0x00)?);
+        let cf_sync = Arc::new(CfSync::new(Arc::clone(&self.headers), skip_blocks, 0x00)?);
 
-        let initial_height = inner.headers.get_height()?;
+        let initial_height = self.headers.get_height()?;
         let total_bundles = (first_peer.get_version().start_height as usize)
             .checked_sub(skip_blocks)
             .map(|x| x / 1000)
             .unwrap_or(0)
             + 1;
-        let expected_bundles_to_sync = total_bundles
-            .checked_sub(cf_sync.pruned_bundles()?)
-            .unwrap_or(0);
+        let expected_bundles_to_sync = total_bundles.saturating_sub(cf_sync.pruned_bundles()?);
 
         let headers_cost = (first_peer.get_version().start_height as usize)
-            .checked_sub(initial_height)
-            .unwrap_or(0) as f32
+            .saturating_sub(initial_height) as f32
             * SYNC_HEADERS_COST;
         let filters_cost = expected_bundles_to_sync as f32 * SYNC_FILTERS_COST;
 
@@ -297,26 +256,24 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
 
         if let Some(snapshot) = sync::sync_headers(
             Arc::clone(&first_peer),
-            Arc::clone(&inner.headers),
+            Arc::clone(&self.headers),
             |new_height| {
                 let local_headers_cost =
-                    new_height.checked_sub(initial_height).unwrap_or(0) as f32 * SYNC_HEADERS_COST;
+                    new_height.saturating_sub(initial_height) as f32 * SYNC_HEADERS_COST;
                 progress_update.update(
                     local_headers_cost / total_cost * 100.0,
                     Some(format!("Synced headers to {}", new_height)),
                 )
             },
         )? {
-            if snapshot.work()? > inner.headers.work()? {
+            if snapshot.work()? > self.headers.work()? {
                 info!("Applying snapshot with work: {}", snapshot.work()?);
-                inner.headers.apply_snapshot(snapshot)?;
+                self.headers.apply_snapshot(snapshot)?;
             }
         }
 
-        let synced_height = inner.headers.get_height()?;
-        let buried_height = synced_height
-            .checked_sub(sync::BURIED_CONFIRMATIONS)
-            .unwrap_or(0);
+        let synced_height = self.headers.get_height()?;
+        let buried_height = synced_height.saturating_sub(sync::BURIED_CONFIRMATIONS);
         info!("Synced headers to height: {}", synced_height);
 
         cf_sync.prepare_sync(Arc::clone(&first_peer))?;
@@ -329,15 +286,17 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
                 .collect::<Vec<_>>(),
         );
 
+        #[allow(clippy::mutex_atomic)]
         let last_synced_block = Arc::new(Mutex::new(synced_height));
+
         let synced_bundles = Arc::new(AtomicUsize::new(0));
         let progress_update = Arc::new(Mutex::new(progress_update));
 
-        let mut threads = Vec::with_capacity(inner.peers.len());
-        for peer in &inner.peers {
+        let mut threads = Vec::with_capacity(self.peers.len());
+        for peer in &self.peers {
             let cf_sync = Arc::clone(&cf_sync);
             let peer = Arc::clone(&peer);
-            let headers = Arc::clone(&inner.headers);
+            let headers = Arc::clone(&self.headers);
             let all_scripts = Arc::clone(&all_scripts);
             let last_synced_block = Arc::clone(&last_synced_block);
             let progress_update = Arc::clone(&progress_update);
@@ -354,10 +313,7 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
                         }
 
                         let block_height = headers.get_height_for(block_hash)?.unwrap_or(0);
-                        let saved_correct_block = match headers.get_full_block(block_height)? {
-                            Some(block) if &block.block_hash() == block_hash => true,
-                            _ => false,
-                        };
+                        let saved_correct_block = matches!(headers.get_full_block(block_height)?, Some(block) if &block.block_hash() == block_hash);
 
                         if saved_correct_block {
                             Ok(false)
@@ -408,57 +364,66 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
         );
         let mut updates = database.begin_batch();
         for details in database.iter_txs(false)? {
-            match details.height {
-                Some(height) if (height as usize) < last_synced_block => continue,
+            match details.confirmation_time {
+                Some(c) if (c.height as usize) < last_synced_block => continue,
                 _ => updates.del_tx(&details.txid, false)?,
             };
         }
         database.commit_batch(updates)?;
 
-        first_peer.ask_for_mempool()?;
+        match first_peer.ask_for_mempool() {
+            Err(CompactFiltersError::PeerBloomDisabled) => {
+                log::warn!("Peer has BLOOM disabled, we can't ask for the mempool")
+            }
+            e => e?,
+        };
 
         let mut internal_max_deriv = None;
         let mut external_max_deriv = None;
 
-        for (height, block) in inner.headers.iter_full_blocks()? {
+        for (height, block) in self.headers.iter_full_blocks()? {
             for tx in &block.txdata {
-                inner.process_tx(
+                self.process_tx(
                     database,
                     tx,
                     Some(height as u32),
-                    0,
+                    None,
                     &mut internal_max_deriv,
                     &mut external_max_deriv,
                 )?;
             }
         }
         for tx in first_peer.get_mempool().iter_txs().iter() {
-            inner.process_tx(
+            self.process_tx(
                 database,
                 tx,
                 None,
-                0,
+                None,
                 &mut internal_max_deriv,
                 &mut external_max_deriv,
             )?;
         }
 
-        let current_ext = database.get_last_index(ScriptType::External)?.unwrap_or(0);
+        let current_ext = database
+            .get_last_index(KeychainKind::External)?
+            .unwrap_or(0);
         let first_ext_new = external_max_deriv.map(|x| x + 1).unwrap_or(0);
         if first_ext_new > current_ext {
             info!("Setting external index to {}", first_ext_new);
-            database.set_last_index(ScriptType::External, first_ext_new)?;
+            database.set_last_index(KeychainKind::External, first_ext_new)?;
         }
 
-        let current_int = database.get_last_index(ScriptType::Internal)?.unwrap_or(0);
+        let current_int = database
+            .get_last_index(KeychainKind::Internal)?
+            .unwrap_or(0);
         let first_int_new = internal_max_deriv.map(|x| x + 1).unwrap_or(0);
         if first_int_new > current_int {
             info!("Setting internal index to {}", first_int_new);
-            database.set_last_index(ScriptType::Internal, first_int_new)?;
+            database.set_last_index(KeychainKind::Internal, first_int_new)?;
         }
 
         info!("Dropping blocks until {}", buried_height);
-        inner.headers.delete_blocks_until(buried_height)?;
+        self.headers.delete_blocks_until(buried_height)?;
 
         progress_update
             .lock()
@@ -469,24 +434,19 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-
-        Ok(inner.peers[0]
+        Ok(self.peers[0]
             .get_mempool()
             .get_tx(&Inventory::Transaction(*txid)))
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-        inner.peers[0].broadcast_tx(tx.clone())?;
+        self.peers[0].broadcast_tx(tx.clone())?;
 
         Ok(())
     }
 
     fn get_height(&self) -> Result<u32, Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-
-        Ok(inner.headers.get_height()? as u32)
+        Ok(self.headers.get_height()? as u32)
     }
 
     fn estimate_fee(&self, _target: usize) -> Result<FeeRate, Error> {
@@ -495,6 +455,61 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
     }
 }
 
+/// Data to connect to a Bitcoin P2P peer
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
+pub struct BitcoinPeerConfig {
+    /// Peer address such as 127.0.0.1:18333
+    pub address: String,
+    /// Optional socks5 proxy
+    pub socks5: Option<String>,
+    /// Optional socks5 proxy credentials
+    pub socks5_credentials: Option<(String, String)>,
+}
+
+/// Configuration for a [`CompactFiltersBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
+pub struct CompactFiltersBlockchainConfig {
+    /// List of peers to try to connect to for asking headers and filters
+    pub peers: Vec<BitcoinPeerConfig>,
+    /// Network used
+    pub network: Network,
+    /// Storage dir to save partially downloaded headers and full blocks
+    pub storage_dir: String,
+    /// Optionally skip initial `skip_blocks` blocks (default: 0)
+    pub skip_blocks: Option<usize>,
+}
+
+impl ConfigurableBlockchain for CompactFiltersBlockchain {
+    type Config = CompactFiltersBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let mempool = Arc::new(Mempool::default());
+        let peers = config
+            .peers
+            .iter()
+            .map(|peer_conf| match &peer_conf.socks5 {
+                None => Peer::connect(&peer_conf.address, Arc::clone(&mempool), config.network),
+                Some(proxy) => Peer::connect_proxy(
+                    peer_conf.address.as_str(),
+                    proxy,
+                    peer_conf
+                        .socks5_credentials
+                        .as_ref()
+                        .map(|(a, b)| (a.as_str(), b.as_str())),
+                    Arc::clone(&mempool),
+                    config.network,
+                ),
+            })
+            .collect::<Result<_, _>>()?;
+
+        Ok(CompactFiltersBlockchain::new(
+            peers,
+            &config.storage_dir,
+            config.skip_blocks,
+        )?)
+    }
+}
+
 /// An error that can occur during sync with a [`CompactFiltersBlockchain`]
 #[derive(Debug)]
 pub enum CompactFiltersError {
@@ -515,16 +530,18 @@ pub enum CompactFiltersError {
     NotConnected,
     /// A peer took too long to reply to one of our messages
     Timeout,
+    /// The peer doesn't advertise the [`BLOOM`](bitcoin::network::constants::ServiceFlags::BLOOM) service flag
+    PeerBloomDisabled,
 
     /// No peers have been specified
     NoPeers,
 
     /// Internal database error
-    DB(rocksdb::Error),
+    Db(rocksdb::Error),
     /// Internal I/O error
-    IO(std::io::Error),
+    Io(std::io::Error),
     /// Invalid BIP158 filter
-    BIP158(bitcoin::util::bip158::Error),
+    Bip158(bitcoin::util::bip158::Error),
     /// Internal system time error
     Time(std::time::SystemTimeError),
 
@@ -540,20 +557,10 @@ impl fmt::Display for CompactFiltersError {
 
 impl std::error::Error for CompactFiltersError {}
 
-macro_rules! impl_error {
-    ( $from:ty, $to:ident ) => {
-        impl std::convert::From<$from> for CompactFiltersError {
-            fn from(err: $from) -> Self {
-                CompactFiltersError::$to(err)
-            }
-        }
-    };
-}
-
-impl_error!(rocksdb::Error, DB);
-impl_error!(std::io::Error, IO);
-impl_error!(bitcoin::util::bip158::Error, BIP158);
-impl_error!(std::time::SystemTimeError, Time);
+impl_error!(rocksdb::Error, Db, CompactFiltersError);
+impl_error!(std::io::Error, Io, CompactFiltersError);
+impl_error!(bitcoin::util::bip158::Error, Bip158, CompactFiltersError);
+impl_error!(std::time::SystemTimeError, Time, CompactFiltersError);
 
 impl From<crate::error::Error> for CompactFiltersError {
     fn from(err: crate::error::Error) -> Self {

src/blockchain/compact_filters/peer.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::collections::HashMap;
 use std::net::{TcpStream, ToSocketAddrs};
@@ -34,7 +21,6 @@ use rand::{thread_rng, Rng};
 
 use bitcoin::consensus::Encodable;
 use bitcoin::hash_types::BlockHash;
-use bitcoin::hashes::Hash;
 use bitcoin::network::constants::ServiceFlags;
 use bitcoin::network::message::{NetworkMessage, RawNetworkMessage};
 use bitcoin::network::message_blockdata::*;
@@ -42,7 +28,7 @@ use bitcoin::network::message_filter::*;
 use bitcoin::network::message_network::VersionMessage;
 use bitcoin::network::stream_reader::StreamReader;
 use bitcoin::network::Address;
-use bitcoin::{Block, Network, Transaction, Txid};
+use bitcoin::{Block, Network, Transaction, Txid, Wtxid};
 
 use super::CompactFiltersError;
 
@@ -55,37 +41,71 @@ pub(crate) const TIMEOUT_SECS: u64 = 30;
 /// It is normally shared between [`Peer`]s with the use of [`Arc`], so that transactions are not
 /// duplicated in memory.
 #[derive(Debug, Default)]
-pub struct Mempool {
-    txs: RwLock<HashMap<Txid, Transaction>>,
+pub struct Mempool(RwLock<InnerMempool>);
+
+#[derive(Debug, Default)]
+struct InnerMempool {
+    txs: HashMap<Txid, Transaction>,
+    wtxids: HashMap<Wtxid, Txid>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum TxIdentifier {
+    Wtxid(Wtxid),
+    Txid(Txid),
 }
 
 impl Mempool {
+    /// Create a new empty mempool
+    pub fn new() -> Self {
+        Self::default()
+    }
+
     /// Add a transaction to the mempool
     ///
     /// Note that this doesn't propagate the transaction to other
-    /// peers. To do that, [`broadcast`](crate::blockchain::OnlineBlockchain::broadcast) should be used.
+    /// peers. To do that, [`broadcast`](crate::blockchain::Blockchain::broadcast) should be used.
     pub fn add_tx(&self, tx: Transaction) {
-        self.txs.write().unwrap().insert(tx.txid(), tx);
+        let mut guard = self.0.write().unwrap();
+
+        guard.wtxids.insert(tx.wtxid(), tx.txid());
+        guard.txs.insert(tx.txid(), tx);
     }
 
     /// Look-up a transaction in the mempool given an [`Inventory`] request
     pub fn get_tx(&self, inventory: &Inventory) -> Option<Transaction> {
-        let txid = match inventory {
+        let identifer = match inventory {
             Inventory::Error | Inventory::Block(_) | Inventory::WitnessBlock(_) => return None,
-            Inventory::Transaction(txid) => *txid,
-            Inventory::WitnessTransaction(wtxid) => Txid::from_inner(wtxid.into_inner()),
+            Inventory::Transaction(txid) => TxIdentifier::Txid(*txid),
+            Inventory::WitnessTransaction(txid) => TxIdentifier::Txid(*txid),
+            Inventory::WTx(wtxid) => TxIdentifier::Wtxid(*wtxid),
+            Inventory::Unknown { inv_type, hash } => {
+                log::warn!(
+                    "Unknown inventory request type `{}`, hash `{:?}`",
+                    inv_type,
+                    hash
+                );
+                return None;
+            }
         };
-        self.txs.read().unwrap().get(&txid).cloned()
+
+        let txid = match identifer {
+            TxIdentifier::Txid(txid) => Some(txid),
+            TxIdentifier::Wtxid(wtxid) => self.0.read().unwrap().wtxids.get(&wtxid).cloned(),
+        };
+
+        txid.map(|txid| self.0.read().unwrap().txs.get(&txid).cloned())
+            .flatten()
     }
 
     /// Return whether or not the mempool contains a transaction with a given txid
     pub fn has_tx(&self, txid: &Txid) -> bool {
-        self.txs.read().unwrap().contains_key(txid)
+        self.0.read().unwrap().txs.contains_key(txid)
     }
 
     /// Return the list of transactions contained in the mempool
     pub fn iter_txs(&self) -> Vec<Transaction> {
-        self.txs.read().unwrap().values().cloned().collect()
+        self.0.read().unwrap().txs.values().cloned().collect()
     }
 }
 
@@ -108,7 +128,7 @@ impl Peer {
     /// Connect to a peer over a plaintext TCP connection
     ///
     /// This function internally spawns a new thread that will monitor incoming messages from the
-    /// peer, and optionally reply to some of them transparently, like [pings](NetworkMessage::Ping)
+    /// peer, and optionally reply to some of them transparently, like [pings](bitcoin::network::message::NetworkMessage::Ping)
     pub fn connect<A: ToSocketAddrs>(
         address: A,
         mempool: Arc<Mempool>,
@@ -190,14 +210,14 @@ impl Peer {
             )),
         )?;
         let version = if let NetworkMessage::Version(version) =
-            Self::_recv(&responses, "version", None)?.unwrap()
+            Self::_recv(&responses, "version", None).unwrap()
         {
             version
         } else {
             return Err(CompactFiltersError::InvalidResponse);
         };
 
-        if let NetworkMessage::Verack = Self::_recv(&responses, "verack", None)?.unwrap() {
+        if let NetworkMessage::Verack = Self::_recv(&responses, "verack", None).unwrap() {
             Self::_send(&mut locked_writer, network.magic(), NetworkMessage::Verack)?;
         } else {
             return Err(CompactFiltersError::InvalidResponse);
@@ -207,12 +227,12 @@ impl Peer {
 
         Ok(Peer {
             writer,
-            reader_thread,
             responses,
+            reader_thread,
             connected,
             mempool,
-            network,
             version,
+            network,
         })
     }
 
@@ -238,7 +258,7 @@ impl Peer {
         responses: &Arc<RwLock<ResponsesMap>>,
         wait_for: &'static str,
         timeout: Option<Duration>,
-    ) -> Result<Option<NetworkMessage>, CompactFiltersError> {
+    ) -> Option<NetworkMessage> {
         let message_resp = {
             let mut lock = responses.write().unwrap();
             let message_resp = lock.entry(wait_for).or_default();
@@ -254,15 +274,14 @@ impl Peer {
                 Some(t) => {
                     let result = cvar.wait_timeout(messages, t).unwrap();
                     if result.1.timed_out() {
-                        return Ok(None);
+                        return None;
                     }
-
                     messages = result.0;
                 }
             }
         }
 
-        Ok(messages.pop())
+        messages.pop()
     }
 
     /// Return the [`VersionMessage`] sent by the peer
@@ -333,7 +352,7 @@ impl Peer {
                 NetworkMessage::Alert(_) => continue,
                 NetworkMessage::GetData(ref inv) => {
                     let (found, not_found): (Vec<_>, Vec<_>) = inv
-                        .into_iter()
+                        .iter()
                         .map(|item| (*item, reader_thread_mempool.get_tx(item)))
                         .partition(|(_, d)| d.is_some());
                     for (_, found_tx) in found {
@@ -382,7 +401,7 @@ impl Peer {
         wait_for: &'static str,
         timeout: Option<Duration>,
     ) -> Result<Option<NetworkMessage>, CompactFiltersError> {
-        Self::_recv(&self.responses, wait_for, timeout)
+        Ok(Self::_recv(&self.responses, wait_for, timeout))
     }
 }
 
@@ -508,6 +527,10 @@ impl InvPeer for Peer {
     }
 
     fn ask_for_mempool(&self) -> Result<(), CompactFiltersError> {
+        if !self.version.services.has(ServiceFlags::BLOOM) {
+            return Err(CompactFiltersError::PeerBloomDisabled);
+        }
+
         self.send(NetworkMessage::MemPool)?;
         let inv = match self.recv("inv", Some(Duration::from_secs(5)))? {
             None => return Ok(()), // empty mempool
@@ -518,10 +541,9 @@ impl InvPeer for Peer {
         let getdata = inv
             .iter()
             .cloned()
-            .filter(|item| match item {
-                Inventory::Transaction(txid) if !self.mempool.has_tx(txid) => true,
-                _ => false,
-            })
+            .filter(
+                |item| matches!(item, Inventory::Transaction(txid) if !self.mempool.has_tx(txid)),
+            )
             .collect::<Vec<_>>();
         let num_txs = getdata.len();
         self.send(NetworkMessage::GetData(getdata))?;

src/blockchain/compact_filters/store.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::convert::TryInto;
 use std::fmt;
@@ -36,9 +23,9 @@ use rand::{thread_rng, Rng};
 use rocksdb::{Direction, IteratorMode, ReadOptions, WriteBatch, DB};
 
 use bitcoin::consensus::{deserialize, encode::VarInt, serialize, Decodable, Encodable};
-use bitcoin::hash_types::FilterHash;
+use bitcoin::hash_types::{FilterHash, FilterHeader};
 use bitcoin::hashes::hex::FromHex;
-use bitcoin::hashes::{sha256d, Hash};
+use bitcoin::hashes::Hash;
 use bitcoin::util::bip158::BlockFilter;
 use bitcoin::util::uint::Uint256;
 use bitcoin::Block;
@@ -46,12 +33,15 @@ use bitcoin::BlockHash;
 use bitcoin::BlockHeader;
 use bitcoin::Network;
 
+use lazy_static::lazy_static;
+
 use super::CompactFiltersError;
 
 lazy_static! {
     static ref MAINNET_GENESIS: Block = deserialize(&Vec::<u8>::from_hex("0100000000000000000000000000000000000000000000000000000000000000000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA4B1E5E4A29AB5F49FFFF001D1DAC2B7C0101000000010000000000000000000000000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D0104455468652054696D65732030332F4A616E2F32303039204368616E63656C6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F757420666F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE5548271967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000").unwrap()).unwrap();
     static ref TESTNET_GENESIS: Block = deserialize(&Vec::<u8>::from_hex("0100000000000000000000000000000000000000000000000000000000000000000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA4B1E5E4ADAE5494DFFFF001D1AA4AE180101000000010000000000000000000000000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D0104455468652054696D65732030332F4A616E2F32303039204368616E63656C6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F757420666F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE5548271967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000").unwrap()).unwrap();
     static ref REGTEST_GENESIS: Block = deserialize(&Vec::<u8>::from_hex("0100000000000000000000000000000000000000000000000000000000000000000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA4B1E5E4ADAE5494DFFFF7F20020000000101000000010000000000000000000000000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D0104455468652054696D65732030332F4A616E2F32303039204368616E63656C6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F757420666F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE5548271967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000").unwrap()).unwrap();
+    static ref SIGNET_GENESIS: Block = deserialize(&Vec::<u8>::from_hex("0100000000000000000000000000000000000000000000000000000000000000000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA4B1E5E4A008F4D5FAE77031E8AD222030101000000010000000000000000000000000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D0104455468652054696D65732030332F4A616E2F32303039204368616E63656C6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F757420666F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE5548271967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000").unwrap()).unwrap();
 }
 
 pub trait StoreType: Default + fmt::Debug {}
@@ -118,45 +108,19 @@ where
     }
 
     fn deserialize(data: &[u8]) -> Result<Self, CompactFiltersError> {
-        Ok(deserialize(data).map_err(|_| CompactFiltersError::DataCorruption)?)
-    }
-}
-
-impl Encodable for FilterHeader {
-    fn consensus_encode<W: Write>(
-        &self,
-        mut e: W,
-    ) -> Result<usize, bitcoin::consensus::encode::Error> {
-        let mut written = self.prev_header_hash.consensus_encode(&mut e)?;
-        written += self.filter_hash.consensus_encode(&mut e)?;
-        Ok(written)
-    }
-}
-
-impl Decodable for FilterHeader {
-    fn consensus_decode<D: Read>(mut d: D) -> Result<Self, bitcoin::consensus::encode::Error> {
-        let prev_header_hash = FilterHeaderHash::consensus_decode(&mut d)?;
-        let filter_hash = FilterHash::consensus_decode(&mut d)?;
-
-        Ok(FilterHeader {
-            prev_header_hash,
-            filter_hash,
-        })
+        deserialize(data).map_err(|_| CompactFiltersError::DataCorruption)
     }
 }
 
 impl Encodable for BundleStatus {
-    fn consensus_encode<W: Write>(
-        &self,
-        mut e: W,
-    ) -> Result<usize, bitcoin::consensus::encode::Error> {
+    fn consensus_encode<W: Write>(&self, mut e: W) -> Result<usize, std::io::Error> {
         let mut written = 0;
 
         match self {
             BundleStatus::Init => {
                 written += 0x00u8.consensus_encode(&mut e)?;
             }
-            BundleStatus::CFHeaders { cf_headers } => {
+            BundleStatus::CfHeaders { cf_headers } => {
                 written += 0x01u8.consensus_encode(&mut e)?;
                 written += VarInt(cf_headers.len() as u64).consensus_encode(&mut e)?;
                 for header in cf_headers {
@@ -207,7 +171,7 @@ impl Decodable for BundleStatus {
                     cf_headers.push(FilterHeader::consensus_decode(&mut d)?);
                 }
 
-                Ok(BundleStatus::CFHeaders { cf_headers })
+                Ok(BundleStatus::CfHeaders { cf_headers })
             }
             0x02 => {
                 let num = VarInt::consensus_decode(&mut d)?;
@@ -264,6 +228,7 @@ impl ChainStore<Full> {
             Network::Bitcoin => MAINNET_GENESIS.deref(),
             Network::Testnet => TESTNET_GENESIS.deref(),
             Network::Regtest => REGTEST_GENESIS.deref(),
+            Network::Signet => SIGNET_GENESIS.deref(),
         };
 
         let cf_name = "default".to_string();
@@ -375,7 +340,7 @@ impl ChainStore<Full> {
         let min_height = match iterator
             .next()
             .and_then(|(k, _)| k[1..].try_into().ok())
-            .map(|bytes| usize::from_be_bytes(bytes))
+            .map(usize::from_be_bytes)
         {
             None => {
                 std::mem::drop(iterator);
@@ -444,9 +409,6 @@ impl ChainStore<Full> {
         }
 
         read_store.write(batch)?;
-
-        std::mem::drop(snapshot_cf_handle);
-        std::mem::drop(cf_handle);
         std::mem::drop(read_store);
 
         self.store.write().unwrap().drop_cf(&snaphost.cf_name)?;
@@ -461,17 +423,16 @@ impl ChainStore<Full> {
         let read_store = self.store.read().unwrap();
         let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
 
-        let key = StoreEntry::BlockHeaderIndex(Some(block_hash.clone())).get_key();
+        let key = StoreEntry::BlockHeaderIndex(Some(*block_hash)).get_key();
         let data = read_store.get_pinned_cf(cf_handle, key)?;
-        Ok(data
-            .map(|data| {
-                Ok::<_, CompactFiltersError>(usize::from_be_bytes(
-                    data.as_ref()
-                        .try_into()
-                        .map_err(|_| CompactFiltersError::DataCorruption)?,
-                ))
-            })
-            .transpose()?)
+        data.map(|data| {
+            Ok::<_, CompactFiltersError>(usize::from_be_bytes(
+                data.as_ref()
+                    .try_into()
+                    .map_err(|_| CompactFiltersError::DataCorruption)?,
+            ))
+        })
+        .transpose()
     }
 
     pub fn get_block_hash(&self, height: usize) -> Result<Option<BlockHash>, CompactFiltersError> {
@@ -480,13 +441,12 @@ impl ChainStore<Full> {
 
         let key = StoreEntry::BlockHeader(Some(height)).get_key();
         let data = read_store.get_pinned_cf(cf_handle, key)?;
-        Ok(data
-            .map(|data| {
-                let (header, _): (BlockHeader, Uint256) =
-                    deserialize(&data).map_err(|_| CompactFiltersError::DataCorruption)?;
-                Ok::<_, CompactFiltersError>(header.block_hash())
-            })
-            .transpose()?)
+        data.map(|data| {
+            let (header, _): (BlockHeader, Uint256) =
+                deserialize(&data).map_err(|_| CompactFiltersError::DataCorruption)?;
+            Ok::<_, CompactFiltersError>(header.block_hash())
+        })
+        .transpose()
     }
 
     pub fn save_full_block(&self, block: &Block, height: usize) -> Result<(), CompactFiltersError> {
@@ -502,10 +462,10 @@ impl ChainStore<Full> {
         let key = StoreEntry::Block(Some(height)).get_key();
         let opt_block = read_store.get_pinned(key)?;
 
-        Ok(opt_block
+        opt_block
             .map(|data| deserialize(&data))
             .transpose()
-            .map_err(|_| CompactFiltersError::DataCorruption)?)
+            .map_err(|_| CompactFiltersError::DataCorruption)
     }
 
     pub fn delete_blocks_until(&self, height: usize) -> Result<(), CompactFiltersError> {
@@ -592,14 +552,14 @@ impl<T: StoreType> ChainStore<T> {
         let prefix = StoreEntry::BlockHeader(None).get_key();
         let iterator = read_store.prefix_iterator_cf(cf_handle, prefix);
 
-        Ok(iterator
+        iterator
             .last()
             .map(|(_, v)| -> Result<_, CompactFiltersError> {
                 let (header, _): (BlockHeader, Uint256) = SerializeDb::deserialize(&v)?;
 
                 Ok(header.block_hash())
             })
-            .transpose()?)
+            .transpose()
     }
 
     pub fn apply(
@@ -642,7 +602,6 @@ impl<T: StoreType> ChainStore<T> {
             );
         }
 
-        std::mem::drop(cf_handle);
         std::mem::drop(read_store);
 
         self.store.write().unwrap().write(batch)?;
@@ -662,44 +621,28 @@ impl<T: StoreType> fmt::Debug for ChainStore<T> {
     }
 }
 
-pub type FilterHeaderHash = FilterHash;
-
-#[derive(Debug, Clone)]
-pub struct FilterHeader {
-    prev_header_hash: FilterHeaderHash,
-    filter_hash: FilterHash,
-}
-
-impl FilterHeader {
-    fn header_hash(&self) -> FilterHeaderHash {
-        let mut hash_data = self.filter_hash.into_inner().to_vec();
-        hash_data.extend_from_slice(&self.prev_header_hash);
-        sha256d::Hash::hash(&hash_data).into()
-    }
-}
-
 pub enum BundleStatus {
     Init,
-    CFHeaders { cf_headers: Vec<FilterHeader> },
+    CfHeaders { cf_headers: Vec<FilterHeader> },
     CFilters { cf_filters: Vec<Vec<u8>> },
     Processed { cf_filters: Vec<Vec<u8>> },
     Tip { cf_filters: Vec<Vec<u8>> },
     Pruned,
 }
 
-pub struct CFStore {
+pub struct CfStore {
     store: Arc<RwLock<DB>>,
     filter_type: u8,
 }
 
-type BundleEntry = (BundleStatus, FilterHeaderHash);
+type BundleEntry = (BundleStatus, FilterHeader);
 
-impl CFStore {
+impl CfStore {
     pub fn new(
         headers_store: &ChainStore<Full>,
         filter_type: u8,
     ) -> Result<Self, CompactFiltersError> {
-        let cf_store = CFStore {
+        let cf_store = CfStore {
             store: Arc::clone(&headers_store.store),
             filter_type,
         };
@@ -708,6 +651,7 @@ impl CFStore {
             Network::Bitcoin => MAINNET_GENESIS.deref(),
             Network::Testnet => TESTNET_GENESIS.deref(),
             Network::Regtest => REGTEST_GENESIS.deref(),
+            Network::Signet => SIGNET_GENESIS.deref(),
         };
 
         let filter = BlockFilter::new_script_filter(genesis, |utxo| {
@@ -721,7 +665,11 @@ impl CFStore {
             if read_store.get_pinned(&first_key)?.is_none() {
                 read_store.put(
                     &first_key,
-                    (BundleStatus::Init, filter.filter_id(&FilterHash::default())).serialize(),
+                    (
+                        BundleStatus::Init,
+                        filter.filter_header(&FilterHeader::from_hash(Default::default())),
+                    )
+                        .serialize(),
                 )?;
             }
         }
@@ -747,7 +695,7 @@ impl CFStore {
             .collect::<Result<_, _>>()
     }
 
-    pub fn get_checkpoints(&self) -> Result<Vec<FilterHash>, CompactFiltersError> {
+    pub fn get_checkpoints(&self) -> Result<Vec<FilterHeader>, CompactFiltersError> {
         let read_store = self.store.read().unwrap();
 
         let prefix = StoreEntry::CFilterTable((self.filter_type, None)).get_key();
@@ -755,16 +703,16 @@ impl CFStore {
 
         // FIXME: we have to filter manually because rocksdb sometimes returns stuff that doesn't
         // have the right prefix
-        Ok(iterator
+        iterator
             .filter(|(k, _)| k.starts_with(&prefix))
             .skip(1)
             .map(|(_, data)| Ok::<_, CompactFiltersError>(BundleEntry::deserialize(&data)?.1))
-            .collect::<Result<_, _>>()?)
+            .collect::<Result<_, _>>()
     }
 
     pub fn replace_checkpoints(
         &self,
-        checkpoints: Vec<FilterHash>,
+        checkpoints: Vec<FilterHeader>,
     ) -> Result<(), CompactFiltersError> {
         let current_checkpoints = self.get_checkpoints()?;
 
@@ -806,20 +754,16 @@ impl CFStore {
     pub fn advance_to_cf_headers(
         &self,
         bundle: usize,
-        checkpoint_hash: FilterHeaderHash,
-        filter_headers: Vec<FilterHash>,
+        checkpoint: FilterHeader,
+        filter_hashes: Vec<FilterHash>,
     ) -> Result<BundleStatus, CompactFiltersError> {
-        let mut last_hash = checkpoint_hash;
-        let cf_headers = filter_headers
+        let cf_headers: Vec<FilterHeader> = filter_hashes
             .into_iter()
-            .map(|filter_hash| {
-                let filter_header = FilterHeader {
-                    prev_header_hash: last_hash,
-                    filter_hash,
-                };
-                last_hash = filter_header.header_hash();
+            .scan(checkpoint, |prev_header, filter_hash| {
+                let filter_header = filter_hash.filter_header(&prev_header);
+                *prev_header = filter_header;
 
-                filter_header
+                Some(filter_header)
             })
             .collect();
 
@@ -832,13 +776,13 @@ impl CFStore {
             .transpose()?
         {
             // check connection with the next bundle if present
-            if last_hash != next_checkpoint {
+            if cf_headers.iter().last() != Some(&next_checkpoint) {
                 return Err(CompactFiltersError::InvalidFilterHeader);
             }
         }
 
         let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
-        let value = (BundleStatus::CFHeaders { cf_headers }, checkpoint_hash);
+        let value = (BundleStatus::CfHeaders { cf_headers }, checkpoint);
 
         read_store.put(key, value.serialize())?;
 
@@ -848,24 +792,26 @@ impl CFStore {
     pub fn advance_to_cf_filters(
         &self,
         bundle: usize,
-        checkpoint_hash: FilterHeaderHash,
+        checkpoint: FilterHeader,
         headers: Vec<FilterHeader>,
         filters: Vec<(usize, Vec<u8>)>,
     ) -> Result<BundleStatus, CompactFiltersError> {
         let cf_filters = filters
             .into_iter()
-            .zip(headers.iter())
-            .map(|((_, filter_content), header)| {
-                if header.filter_hash != sha256d::Hash::hash(&filter_content).into() {
-                    return Err(CompactFiltersError::InvalidFilter);
+            .zip(headers.into_iter())
+            .scan(checkpoint, |prev_header, ((_, filter_content), header)| {
+                let filter = BlockFilter::new(&filter_content);
+                if header != filter.filter_header(&prev_header) {
+                    return Some(Err(CompactFiltersError::InvalidFilter));
                 }
+                *prev_header = header;
 
-                Ok::<_, CompactFiltersError>(filter_content)
+                Some(Ok::<_, CompactFiltersError>(filter_content))
             })
             .collect::<Result<_, _>>()?;
 
         let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
-        let value = (BundleStatus::CFilters { cf_filters }, checkpoint_hash);
+        let value = (BundleStatus::CFilters { cf_filters }, checkpoint);
 
         let read_store = self.store.read().unwrap();
         read_store.put(key, value.serialize())?;
@@ -876,10 +822,10 @@ impl CFStore {
     pub fn prune_filters(
         &self,
         bundle: usize,
-        checkpoint_hash: FilterHeaderHash,
+        checkpoint: FilterHeader,
     ) -> Result<BundleStatus, CompactFiltersError> {
         let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
-        let value = (BundleStatus::Pruned, checkpoint_hash);
+        let value = (BundleStatus::Pruned, checkpoint);
 
         let read_store = self.store.read().unwrap();
         read_store.put(key, value.serialize())?;
@@ -891,10 +837,10 @@ impl CFStore {
         &self,
         bundle: usize,
         cf_filters: Vec<Vec<u8>>,
-        checkpoint_hash: FilterHeaderHash,
+        checkpoint: FilterHeader,
     ) -> Result<BundleStatus, CompactFiltersError> {
         let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
-        let value = (BundleStatus::Tip { cf_filters }, checkpoint_hash);
+        let value = (BundleStatus::Tip { cf_filters }, checkpoint);
 
         let read_store = self.store.read().unwrap();
         read_store.put(key, value.serialize())?;

src/blockchain/compact_filters/sync.rs

@@ -1,32 +1,19 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::collections::{BTreeMap, HashMap, VecDeque};
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use bitcoin::hash_types::{BlockHash, FilterHash};
+use bitcoin::hash_types::{BlockHash, FilterHeader};
 use bitcoin::network::message::NetworkMessage;
 use bitcoin::network::message_blockdata::GetHeadersMessage;
 use bitcoin::util::bip158::BlockFilter;
@@ -38,22 +25,22 @@ use crate::error::Error;
 
 pub(crate) const BURIED_CONFIRMATIONS: usize = 100;
 
-pub struct CFSync {
+pub struct CfSync {
     headers_store: Arc<ChainStore<Full>>,
-    cf_store: Arc<CFStore>,
+    cf_store: Arc<CfStore>,
     skip_blocks: usize,
-    bundles: Mutex<VecDeque<(BundleStatus, FilterHash, usize)>>,
+    bundles: Mutex<VecDeque<(BundleStatus, FilterHeader, usize)>>,
 }
 
-impl CFSync {
+impl CfSync {
     pub fn new(
         headers_store: Arc<ChainStore<Full>>,
         skip_blocks: usize,
         filter_type: u8,
     ) -> Result<Self, CompactFiltersError> {
-        let cf_store = Arc::new(CFStore::new(&headers_store, filter_type)?);
+        let cf_store = Arc::new(CfStore::new(&headers_store, filter_type)?);
 
-        Ok(CFSync {
+        Ok(CfSync {
             headers_store,
             cf_store,
             skip_blocks,
@@ -148,7 +135,7 @@ impl CFSync {
 
                 let resp = peer.get_cf_headers(0x00, start_height as u32, stop_hash)?;
 
-                assert!(resp.previous_filter == checkpoint);
+                assert!(resp.previous_filter_header == checkpoint);
                 status =
                     self.cf_store
                         .advance_to_cf_headers(index, checkpoint, resp.filter_hashes)?;
@@ -164,7 +151,7 @@ impl CFSync {
                     checkpoint,
                     headers_resp.filter_hashes,
                 )? {
-                    BundleStatus::CFHeaders { cf_headers } => cf_headers,
+                    BundleStatus::CfHeaders { cf_headers } => cf_headers,
                     _ => return Err(CompactFiltersError::InvalidResponse),
                 };
 
@@ -184,7 +171,7 @@ impl CFSync {
                     .cf_store
                     .advance_to_cf_filters(index, checkpoint, cf_headers, filters)?;
             }
-            if let BundleStatus::CFHeaders { cf_headers } = status {
+            if let BundleStatus::CfHeaders { cf_headers } = status {
                 log::trace!("status: CFHeaders");
 
                 peer.get_cf_filters(
@@ -204,9 +191,8 @@ impl CFSync {
             if let BundleStatus::CFilters { cf_filters } = status {
                 log::trace!("status: CFilters");
 
-                let last_sync_buried_height = (start_height + already_processed)
-                    .checked_sub(BURIED_CONFIRMATIONS)
-                    .unwrap_or(0);
+                let last_sync_buried_height =
+                    (start_height + already_processed).saturating_sub(BURIED_CONFIRMATIONS);
 
                 for (filter_index, filter) in cf_filters.iter().enumerate() {
                     let height = filter_index + start_height;
@@ -280,10 +266,7 @@ where
 
         match locators_map.get(&headers[0].prev_blockhash) {
             None => return Err(CompactFiltersError::InvalidHeaders),
-            Some(from) => (
-                store.start_snapshot(*from)?,
-                headers[0].prev_blockhash.clone(),
-            ),
+            Some(from) => (store.start_snapshot(*from)?, headers[0].prev_blockhash),
         }
     } else {
         return Err(CompactFiltersError::InvalidResponse);

src/blockchain/electrum.rs

@@ -1,40 +1,27 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Electrum
 //!
-//! This module defines an [`OnlineBlockchain`] struct that wraps an [`electrum_client::Client`]
+//! This module defines a [`Blockchain`] struct that wraps an [`electrum_client::Client`]
 //! and implements the logic required to populate the wallet's [database](crate::database::Database) by
 //! querying the inner client.
 //!
 //! ## Example
 //!
 //! ```no_run
-//! # use magical::blockchain::electrum::ElectrumBlockchain;
-//! let client = electrum_client::Client::new("ssl://electrum.blockstream.info:50002", None)?;
+//! # use bdk::blockchain::electrum::ElectrumBlockchain;
+//! let client = electrum_client::Client::new("ssl://electrum.blockstream.info:50002")?;
 //! let blockchain = ElectrumBlockchain::from(client);
-//! # Ok::<(), magical::Error>(())
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::collections::HashSet;
@@ -42,11 +29,11 @@ use std::collections::HashSet;
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
 
-use electrum_client::{Client, ElectrumApi};
+use electrum_client::{Client, ConfigBuilder, ElectrumApi, Socks5Config};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ElectrumLikeSync, ElsGetHistoryRes};
 use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
@@ -56,32 +43,15 @@ use crate::FeeRate;
 ///
 /// ## Example
 /// See the [`blockchain::electrum`](crate::blockchain::electrum) module for a usage example.
-pub struct ElectrumBlockchain(Option<Client>);
-
-#[cfg(test)]
-#[cfg(feature = "test-electrum")]
-#[magical_blockchain_tests(crate)]
-fn local_electrs() -> ElectrumBlockchain {
-    ElectrumBlockchain::from(Client::new(&testutils::get_electrum_url(), None).unwrap())
-}
+pub struct ElectrumBlockchain(Client);
 
 impl std::convert::From<Client> for ElectrumBlockchain {
     fn from(client: Client) -> Self {
-        ElectrumBlockchain(Some(client))
+        ElectrumBlockchain(client)
     }
 }
 
 impl Blockchain for ElectrumBlockchain {
-    fn offline() -> Self {
-        ElectrumBlockchain(None)
-    }
-
-    fn is_online(&self) -> bool {
-        self.0.is_some()
-    }
-}
-
-impl OnlineBlockchain for ElectrumBlockchain {
     fn get_capabilities(&self) -> HashSet<Capability> {
         vec![
             Capability::FullHistory,
@@ -99,27 +69,15 @@ impl OnlineBlockchain for ElectrumBlockchain {
         progress_update: P,
     ) -> Result<(), Error> {
         self.0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
             .electrum_like_setup(stop_gap, database, progress_update)
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            .transaction_get(txid)
-            .map(Option::Some)?)
+        Ok(self.0.transaction_get(txid).map(Option::Some)?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            .transaction_broadcast(tx)
-            .map(|_| ())?)
+        Ok(self.0.transaction_broadcast(tx).map(|_| ())?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
@@ -127,27 +85,22 @@ impl OnlineBlockchain for ElectrumBlockchain {
 
         Ok(self
             .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
             .block_headers_subscribe()
             .map(|data| data.height as u32)?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
         Ok(FeeRate::from_btc_per_kvb(
-            self.0
-                .as_ref()
-                .ok_or(Error::OfflineClient)?
-                .estimate_fee(target)? as f32,
+            self.0.estimate_fee(target)? as f32
         ))
     }
 }
 
 impl ElectrumLikeSync for Client {
-    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
         &self,
         scripts: I,
-    ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error> {
         self.batch_script_get_history(scripts)
             .map(|v| {
                 v.into_iter()
@@ -156,7 +109,7 @@ impl ElectrumLikeSync for Client {
                             .map(
                                 |electrum_client::GetHistoryRes {
                                      height, tx_hash, ..
-                                 }| ELSGetHistoryRes {
+                                 }| ElsGetHistoryRes {
                                     height,
                                     tx_hash,
                                 },
@@ -168,35 +121,57 @@ impl ElectrumLikeSync for Client {
             .map_err(Error::Electrum)
     }
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid> + Clone>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
-        self.batch_script_list_unspent(scripts)
-            .map(|v| {
-                v.into_iter()
-                    .map(|v| {
-                        v.into_iter()
-                            .map(
-                                |electrum_client::ListUnspentRes {
-                                     height,
-                                     tx_hash,
-                                     tx_pos,
-                                     ..
-                                 }| ELSListUnspentRes {
-                                    height,
-                                    tx_hash,
-                                    tx_pos,
-                                },
-                            )
-                            .collect()
-                    })
-                    .collect()
-            })
-            .map_err(Error::Electrum)
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
+        self.batch_transaction_get(txids).map_err(Error::Electrum)
     }
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error> {
-        self.transaction_get(txid).map_err(Error::Electrum)
+    fn els_batch_block_header<I: IntoIterator<Item = u32> + Clone>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        self.batch_block_header(heights).map_err(Error::Electrum)
+    }
+}
+
+/// Configuration for an [`ElectrumBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
+pub struct ElectrumBlockchainConfig {
+    /// URL of the Electrum server (such as ElectrumX, Esplora, BWT) may start with `ssl://` or `tcp://` and include a port
+    ///
+    /// eg. `ssl://electrum.blockstream.info:60002`
+    pub url: String,
+    /// URL of the socks5 proxy server or a Tor service
+    pub socks5: Option<String>,
+    /// Request retry count
+    pub retry: u8,
+    /// Request timeout (seconds)
+    pub timeout: Option<u8>,
+}
+
+impl ConfigurableBlockchain for ElectrumBlockchain {
+    type Config = ElectrumBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let socks5 = config.socks5.as_ref().map(Socks5Config::new);
+        let electrum_config = ConfigBuilder::new()
+            .retry(config.retry)
+            .timeout(config.timeout)?
+            .socks5(socks5)?
+            .build();
+
+        Ok(ElectrumBlockchain(Client::from_config(
+            config.url.as_str(),
+            electrum_config,
+        )?))
+    }
+}
+
+#[cfg(feature = "test-blockchains")]
+crate::bdk_blockchain_tests! {
+    fn test_instance() -> ElectrumBlockchain {
+        ElectrumBlockchain::from(Client::new(&testutils::blockchain_tests::get_electrum_url()).unwrap())
     }
 }

src/blockchain/esplora.rs

@@ -1,44 +1,31 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Esplora
 //!
-//! This module defines an [`OnlineBlockchain`] struct that can query an Esplora backend
+//! This module defines a [`Blockchain`] struct that can query an Esplora backend
 //! populate the wallet's [database](crate::database::Database) by
 //!
 //! ## Example
 //!
 //! ```no_run
-//! # use magical::blockchain::esplora::EsploraBlockchain;
-//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api");
-//! # Ok::<(), magical::Error>(())
+//! # use bdk::blockchain::esplora::EsploraBlockchain;
+//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", None);
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::collections::{HashMap, HashSet};
 use std::fmt;
 
-use futures::stream::{self, StreamExt, TryStreamExt};
+use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
@@ -47,23 +34,27 @@ use serde::Deserialize;
 
 use reqwest::{Client, StatusCode};
 
-use bitcoin::consensus::{deserialize, serialize};
-use bitcoin::hashes::hex::ToHex;
+use bitcoin::consensus::{self, deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
 use bitcoin::hashes::{sha256, Hash};
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHash, BlockHeader, Script, Transaction, Txid};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ElectrumLikeSync, ElsGetHistoryRes};
 use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
+use crate::wallet::utils::ChunksIterator;
 use crate::FeeRate;
 
+const DEFAULT_CONCURRENT_REQUESTS: u8 = 4;
+
 #[derive(Debug)]
 struct UrlClient {
     url: String,
     // We use the async client instead of the blocking one because it automatically uses `fetch`
     // when the target platform is wasm32.
     client: Client,
+    concurrency: u8,
 }
 
 /// Structure that implements the logic to sync with Esplora
@@ -71,36 +62,27 @@ struct UrlClient {
 /// ## Example
 /// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
 #[derive(Debug)]
-pub struct EsploraBlockchain(Option<UrlClient>);
+pub struct EsploraBlockchain(UrlClient);
 
 impl std::convert::From<UrlClient> for EsploraBlockchain {
     fn from(url_client: UrlClient) -> Self {
-        EsploraBlockchain(Some(url_client))
+        EsploraBlockchain(url_client)
     }
 }
 
 impl EsploraBlockchain {
     /// Create a new instance of the client from a base URL
-    pub fn new(base_url: &str) -> Self {
-        EsploraBlockchain(Some(UrlClient {
+    pub fn new(base_url: &str, concurrency: Option<u8>) -> Self {
+        EsploraBlockchain(UrlClient {
             url: base_url.to_string(),
             client: Client::new(),
-        }))
-    }
-}
-
-impl Blockchain for EsploraBlockchain {
-    fn offline() -> Self {
-        EsploraBlockchain(None)
-    }
-
-    fn is_online(&self) -> bool {
-        self.0.is_some()
+            concurrency: concurrency.unwrap_or(DEFAULT_CONCURRENT_REQUESTS),
+        })
     }
 }
 
 #[maybe_async]
-impl OnlineBlockchain for EsploraBlockchain {
+impl Blockchain for EsploraBlockchain {
     fn get_capabilities(&self) -> HashSet<Capability> {
         vec![
             Capability::FullHistory,
@@ -119,41 +101,23 @@ impl OnlineBlockchain for EsploraBlockchain {
     ) -> Result<(), Error> {
         maybe_await!(self
             .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
             .electrum_like_setup(stop_gap, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._get_tx(txid))?)
+        Ok(await_or_block!(self.0._get_tx(txid))?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._broadcast(tx))?)
+        Ok(await_or_block!(self.0._broadcast(tx))?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
-        Ok(await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._get_height())?)
+        Ok(await_or_block!(self.0._get_height())?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
-        let estimates = await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._get_fee_estimates())?;
+        let estimates = await_or_block!(self.0._get_fee_estimates())?;
 
         let fee_val = estimates
             .into_iter()
@@ -189,6 +153,39 @@ impl UrlClient {
         Ok(Some(deserialize(&resp.error_for_status()?.bytes().await?)?))
     }
 
+    async fn _get_tx_no_opt(&self, txid: &Txid) -> Result<Transaction, EsploraError> {
+        match self._get_tx(txid).await {
+            Ok(Some(tx)) => Ok(tx),
+            Ok(None) => Err(EsploraError::TransactionNotFound(*txid)),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn _get_header(&self, block_height: u32) -> Result<BlockHeader, EsploraError> {
+        let resp = self
+            .client
+            .get(&format!("{}/block-height/{}", self.url, block_height))
+            .send()
+            .await?;
+
+        if let StatusCode::NOT_FOUND = resp.status() {
+            return Err(EsploraError::HeaderHeightNotFound(block_height));
+        }
+        let bytes = resp.bytes().await?;
+        let hash = std::str::from_utf8(&bytes)
+            .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?;
+
+        let resp = self
+            .client
+            .get(&format!("{}/block/{}/header", self.url, hash))
+            .send()
+            .await?;
+
+        let header = deserialize(&Vec::from_hex(&resp.text().await?)?)?;
+
+        Ok(header)
+    }
+
     async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
         self.client
             .post(&format!("{}/tx", self.url))
@@ -213,7 +210,7 @@ impl UrlClient {
     async fn _script_get_history(
         &self,
         script: &Script,
-    ) -> Result<Vec<ELSGetHistoryRes>, EsploraError> {
+    ) -> Result<Vec<ElsGetHistoryRes>, EsploraError> {
         let mut result = Vec::new();
         let scripthash = Self::script_to_scripthash(script);
 
@@ -230,7 +227,7 @@ impl UrlClient {
                 .json::<Vec<EsploraGetHistory>>()
                 .await?
                 .into_iter()
-                .map(|x| ELSGetHistoryRes {
+                .map(|x| ElsGetHistoryRes {
                     tx_hash: x.txid,
                     height: x.status.block_height.unwrap_or(0) as i32,
                 }),
@@ -264,7 +261,7 @@ impl UrlClient {
 
             debug!("... adding {} confirmed transactions", len);
 
-            result.extend(response.into_iter().map(|x| ELSGetHistoryRes {
+            result.extend(response.into_iter().map(|x| ElsGetHistoryRes {
                 tx_hash: x.txid,
                 height: x.status.block_height.unwrap_or(0) as i32,
             }));
@@ -277,31 +274,6 @@ impl UrlClient {
         Ok(result)
     }
 
-    async fn _script_list_unspent(
-        &self,
-        script: &Script,
-    ) -> Result<Vec<ELSListUnspentRes>, EsploraError> {
-        Ok(self
-            .client
-            .get(&format!(
-                "{}/scripthash/{}/utxo",
-                self.url,
-                Self::script_to_scripthash(script)
-            ))
-            .send()
-            .await?
-            .error_for_status()?
-            .json::<Vec<EsploraListUnspent>>()
-            .await?
-            .into_iter()
-            .map(|x| ELSListUnspentRes {
-                tx_hash: x.txid,
-                height: x.status.block_height.unwrap_or(0),
-                tx_pos: x.vout,
-            })
-            .collect())
-    }
-
     async fn _get_fee_estimates(&self) -> Result<HashMap<String, f64>, EsploraError> {
         Ok(self
             .client
@@ -319,34 +291,61 @@ impl ElectrumLikeSync for UrlClient {
     fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
         &self,
         scripts: I,
-    ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error> {
         let future = async {
-            Ok(stream::iter(scripts)
-                .then(|script| self._script_get_history(&script))
-                .try_collect()
-                .await?)
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(scripts.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for script in chunk {
+                    futs.push(self._script_get_history(&script));
+                }
+                let partial_results: Vec<Vec<ElsGetHistoryRes>> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
         };
 
         await_or_block!(future)
     }
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
         let future = async {
-            Ok(stream::iter(scripts)
-                .then(|script| self._script_list_unspent(&script))
-                .try_collect()
-                .await?)
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(txids.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for txid in chunk {
+                    futs.push(self._get_tx_no_opt(&txid));
+                }
+                let partial_results: Vec<Transaction> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
         };
 
         await_or_block!(future)
     }
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error> {
-        Ok(await_or_block!(self._get_tx(txid))?
-            .ok_or_else(|| EsploraError::TransactionNotFound(*txid))?)
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        let future = async {
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(heights.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for height in chunk {
+                    futs.push(self._get_header(height));
+                }
+                let partial_results: Vec<BlockHeader> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
+        };
+
+        await_or_block!(future)
     }
 }
 
@@ -361,11 +360,26 @@ struct EsploraGetHistory {
     status: EsploraGetHistoryStatus,
 }
 
-#[derive(Deserialize)]
-struct EsploraListUnspent {
-    txid: Txid,
-    vout: usize,
-    status: EsploraGetHistoryStatus,
+/// Configuration for an [`EsploraBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)]
+pub struct EsploraBlockchainConfig {
+    /// Base URL of the esplora service
+    ///
+    /// eg. `https://blockstream.info/api/`
+    pub base_url: String,
+    /// Number of parallel requests sent to the esplora service (default: 4)
+    pub concurrency: Option<u8>,
+}
+
+impl ConfigurableBlockchain for EsploraBlockchain {
+    type Config = EsploraBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(EsploraBlockchain::new(
+            config.base_url.as_str(),
+            config.concurrency,
+        ))
+    }
 }
 
 /// Errors that can happen during a sync with [`EsploraBlockchain`]
@@ -377,9 +391,15 @@ pub enum EsploraError {
     Parsing(std::num::ParseIntError),
     /// Invalid Bitcoin data returned
     BitcoinEncoding(bitcoin::consensus::encode::Error),
+    /// Invalid Hex data returned
+    Hex(bitcoin::hashes::hex::Error),
 
     /// Transaction not found
     TransactionNotFound(Txid),
+    /// Header height not found
+    HeaderHeightNotFound(u32),
+    /// Header hash not found
+    HeaderHashNotFound(BlockHash),
 }
 
 impl fmt::Display for EsploraError {
@@ -390,20 +410,14 @@ impl fmt::Display for EsploraError {
 
 impl std::error::Error for EsploraError {}
 
-impl From<reqwest::Error> for EsploraError {
-    fn from(other: reqwest::Error) -> Self {
-        EsploraError::Reqwest(other)
-    }
-}
+impl_error!(reqwest::Error, Reqwest, EsploraError);
+impl_error!(std::num::ParseIntError, Parsing, EsploraError);
+impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
+impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
 
-impl From<std::num::ParseIntError> for EsploraError {
-    fn from(other: std::num::ParseIntError) -> Self {
-        EsploraError::Parsing(other)
-    }
-}
-
-impl From<bitcoin::consensus::encode::Error> for EsploraError {
-    fn from(other: bitcoin::consensus::encode::Error) -> Self {
-        EsploraError::BitcoinEncoding(other)
+#[cfg(feature = "test-blockchains")]
+crate::bdk_blockchain_tests! {
+    fn test_instance() -> EsploraBlockchain {
+        EsploraBlockchain::new(std::env::var("BDK_ESPLORA_URL").unwrap_or("127.0.0.1:3002".into()).as_str(), None)
     }
 }

src/blockchain/mod.rs

@@ -1,42 +1,20 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Blockchain backends
 //!
 //! This module provides the implementation of a few commonly-used backends like
 //! [Electrum](crate::blockchain::electrum), [Esplora](crate::blockchain::esplora) and
-//! [Compact Filters/Neutrino](crate::blockchain::compact_filters), along with two generalized
-//! traits [`Blockchain`] and [`OnlineBlockchain`] that can be implemented to build customized
-//! backends.
-//!
-//! Types that only implement the [`Blockchain`] trait can be used as backends for [`Wallet`](crate::wallet::Wallet)s, but any
-//! action that requires interacting with the blockchain won't be available ([`Wallet::sync`](crate::wallet::Wallet::sync) and
-//! [`Wallet::broadcast`](crate::wallet::Wallet::broadcast)). This allows the creation of physically air-gapped wallets, that have no
-//! ability to contact the outside world. An example of an offline-only client is [`OfflineBlockchain`].
-//!
-//! Types that also implement [`OnlineBlockchain`] will make the two aforementioned actions
-//! available.
+//! [Compact Filters/Neutrino](crate::blockchain::compact_filters), along with a generalized trait
+//! [`Blockchain`] that can be implemented to build customized backends.
 
 use std::collections::HashSet;
 use std::ops::Deref;
@@ -49,13 +27,28 @@ use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::FeeRate;
 
+#[cfg(any(feature = "electrum", feature = "esplora"))]
 pub(crate) mod utils;
 
+#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+pub mod any;
+#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+pub use any::{AnyBlockchain, AnyBlockchainConfig};
+
 #[cfg(feature = "electrum")]
 #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
 pub mod electrum;
 #[cfg(feature = "electrum")]
 pub use self::electrum::ElectrumBlockchain;
+#[cfg(feature = "electrum")]
+pub use self::electrum::ElectrumBlockchainConfig;
+
+#[cfg(feature = "rpc")]
+pub mod rpc;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcBlockchain;
+#[cfg(feature = "rpc")]
+pub use self::rpc::RpcConfig;
 
 #[cfg(feature = "esplora")]
 #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
@@ -66,10 +59,11 @@ pub use self::esplora::EsploraBlockchain;
 #[cfg(feature = "compact_filters")]
 #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
 pub mod compact_filters;
+
 #[cfg(feature = "compact_filters")]
 pub use self::compact_filters::CompactFiltersBlockchain;
 
-/// Capabilities that can be supported by an [`OnlineBlockchain`] backend
+/// Capabilities that can be supported by a [`Blockchain`] backend
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Capability {
     /// Can recover the full history of a wallet and not only the set of currently spendable UTXOs
@@ -80,56 +74,22 @@ pub enum Capability {
     AccurateFees,
 }
 
-/// Base trait for a blockchain backend
-///
-/// This trait is always required, even for "air-gapped" backends that don't actually make any
-/// external call. Clients that have the ability to make external calls must also implement `OnlineBlockchain`.
-pub trait Blockchain {
-    /// Return whether or not the client has the ability to fullfill requests
-    ///
-    /// This should always be `false` for offline-only types, and can be true for types that also
-    /// implement [`OnlineBlockchain`], if they have the ability to fullfill requests.
-    fn is_online(&self) -> bool;
-
-    /// Create a new instance of the client that is offline-only
-    ///
-    /// For types that also implement [`OnlineBlockchain`], this means creating an instance that
-    /// returns [`Error::OfflineClient`](crate::error::Error::OfflineClient) if any of the "online"
-    /// methods are called.
-    ///
-    /// This is generally implemented by wrapping the client in an [`Option`] that has [`Option::None`] value
-    /// when created with this method, and is [`Option::Some`] if properly instantiated.
-    fn offline() -> Self;
-}
-
-/// Type that only implements [`Blockchain`] and is always offline
-pub struct OfflineBlockchain;
-impl Blockchain for OfflineBlockchain {
-    fn offline() -> Self {
-        OfflineBlockchain
-    }
-
-    fn is_online(&self) -> bool {
-        false
-    }
-}
-
-/// Trait that defines the actions that must be supported by an online [`Blockchain`]
+/// Trait that defines the actions that must be supported by a blockchain backend
 #[maybe_async]
-pub trait OnlineBlockchain: Blockchain {
+pub trait Blockchain {
     /// Return the set of [`Capability`] supported by this backend
     fn get_capabilities(&self) -> HashSet<Capability>;
 
     /// Setup the backend and populate the internal database for the first time
     ///
-    /// This method is the equivalent of [`OnlineBlockchain::sync`], but it's guaranteed to only be
+    /// This method is the equivalent of [`Blockchain::sync`], but it's guaranteed to only be
     /// called once, at the first [`Wallet::sync`](crate::wallet::Wallet::sync).
     ///
     /// The rationale behind the distinction between `sync` and `setup` is that some custom backends
     /// might need to perform specific actions only the first time they are synced.
     ///
     /// For types that do not have that distinction, only this method can be implemented, since
-    /// [`OnlineBlockchain::sync`] defaults to calling this internally if not overridden.
+    /// [`Blockchain::sync`] defaults to calling this internally if not overridden.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
         stop_gap: Option<usize>,
@@ -138,7 +98,7 @@ pub trait OnlineBlockchain: Blockchain {
     ) -> Result<(), Error>;
     /// Populate the internal database with transactions and UTXOs
     ///
-    /// If not overridden, it defaults to calling [`OnlineBlockchain::setup`] internally.
+    /// If not overridden, it defaults to calling [`Blockchain::setup`] internally.
     ///
     /// This method should implement the logic required to iterate over the list of the wallet's
     /// script_pubkeys using [`Database::iter_script_pubkeys`] and look for relevant transactions
@@ -175,11 +135,20 @@ pub trait OnlineBlockchain: Blockchain {
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error>;
 }
 
+/// Trait for [`Blockchain`] types that can be created given a configuration
+pub trait ConfigurableBlockchain: Blockchain + Sized {
+    /// Type that contains the configuration
+    type Config: std::fmt::Debug;
+
+    /// Create a new instance given a configuration
+    fn from_config(config: &Self::Config) -> Result<Self, Error>;
+}
+
 /// Data sent with a progress update over a [`channel`]
 pub type ProgressData = (f32, Option<String>);
 
-/// Trait for types that can receive and process progress updates during [`OnlineBlockchain::sync`] and
-/// [`OnlineBlockchain::setup`]
+/// Trait for types that can receive and process progress updates during [`Blockchain::sync`] and
+/// [`Blockchain::setup`]
 pub trait Progress: Send {
     /// Send a new progress update
     ///
@@ -195,7 +164,7 @@ pub fn progress() -> (Sender<ProgressData>, Receiver<ProgressData>) {
 
 impl Progress for Sender<ProgressData> {
     fn update(&self, progress: f32, message: Option<String>) -> Result<(), Error> {
-        if progress < 0.0 || progress > 100.0 {
+        if !(0.0..=100.0).contains(&progress) {
             return Err(Error::InvalidProgressValue(progress));
         }
 
@@ -205,7 +174,7 @@ impl Progress for Sender<ProgressData> {
 }
 
 /// Type that implements [`Progress`] and drops every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct NoopProgress;
 
 /// Create a new instance of [`NoopProgress`]
@@ -220,7 +189,7 @@ impl Progress for NoopProgress {
 }
 
 /// Type that implements [`Progress`] and logs at level `INFO` every update received
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub struct LogProgress;
 
 /// Create a nwe instance of [`LogProgress`]
@@ -230,24 +199,18 @@ pub fn log_progress() -> LogProgress {
 
 impl Progress for LogProgress {
     fn update(&self, progress: f32, message: Option<String>) -> Result<(), Error> {
-        log::info!("Sync {:.3}%: `{}`", progress, message.unwrap_or("".into()));
+        log::info!(
+            "Sync {:.3}%: `{}`",
+            progress,
+            message.unwrap_or_else(|| "".into())
+        );
 
         Ok(())
     }
 }
 
-impl<T: Blockchain> Blockchain for Arc<T> {
-    fn is_online(&self) -> bool {
-        self.deref().is_online()
-    }
-
-    fn offline() -> Self {
-        Arc::new(T::offline())
-    }
-}
-
 #[maybe_async]
-impl<T: OnlineBlockchain> OnlineBlockchain for Arc<T> {
+impl<T: Blockchain> Blockchain for Arc<T> {
     fn get_capabilities(&self) -> HashSet<Capability> {
         maybe_await!(self.deref().get_capabilities())
     }

src/blockchain/rpc.rs

@@ -0,0 +1,676 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Riccardo Casatta <riccardo@casatta.it>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Rpc Blockchain
+//!
+//! Backend that gets blockchain data from Bitcoin Core RPC
+//!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::{RpcConfig, RpcBlockchain, ConfigurableBlockchain};
+//! let config = RpcConfig {
+//!     url: "127.0.0.1:18332".to_string(),
+//!     auth: bitcoincore_rpc::Auth::CookieFile("/home/user/.bitcoin/.cookie".into()),
+//!     network: bdk::bitcoin::Network::Testnet,
+//!     wallet_name: "wallet_name".to_string(),
+//!     skip_blocks: None,
+//! };
+//! let blockchain = RpcBlockchain::from_config(&config);
+//! ```
+
+use crate::bitcoin::consensus::deserialize;
+use crate::bitcoin::{Address, Network, OutPoint, Transaction, TxOut, Txid};
+use crate::blockchain::{Blockchain, Capability, ConfigurableBlockchain, Progress};
+use crate::database::{BatchDatabase, DatabaseUtils};
+use crate::descriptor::{get_checksum, IntoWalletDescriptor};
+use crate::wallet::utils::SecpCtx;
+use crate::{ConfirmationTime, Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails};
+use bitcoincore_rpc::json::{
+    GetAddressInfoResultLabel, ImportMultiOptions, ImportMultiRequest,
+    ImportMultiRequestScriptPubkey, ImportMultiRescanSince,
+};
+use bitcoincore_rpc::jsonrpc::serde_json::Value;
+use bitcoincore_rpc::{Auth, Client, RpcApi};
+use log::debug;
+use serde::Deserialize;
+use std::collections::{HashMap, HashSet};
+use std::str::FromStr;
+
+/// The main struct for RPC backend implementing the [crate::blockchain::Blockchain] trait
+#[derive(Debug)]
+pub struct RpcBlockchain {
+    /// Rpc client to the node, includes the wallet name
+    client: Client,
+    /// Network used
+    network: Network,
+    /// Blockchain capabilities, cached here at startup
+    capabilities: HashSet<Capability>,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    skip_blocks: Option<u32>,
+
+    /// This is a fixed Address used as a hack key to store information on the node
+    _storage_address: Address,
+}
+
+/// RpcBlockchain configuration options
+#[derive(Debug)]
+pub struct RpcConfig {
+    /// The bitcoin node url
+    pub url: String,
+    /// The bitcoin node authentication mechanism
+    pub auth: Auth,
+    /// The network we are using (it will be checked the bitcoin node network matches this)
+    pub network: Network,
+    /// The wallet name in the bitcoin node, consider using [wallet_name_from_descriptor] for this
+    pub wallet_name: String,
+    /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block
+    pub skip_blocks: Option<u32>,
+}
+
+impl RpcBlockchain {
+    fn get_node_synced_height(&self) -> Result<u32, Error> {
+        let info = self.client.get_address_info(&self._storage_address)?;
+        if let Some(GetAddressInfoResultLabel::Simple(label)) = info.labels.first() {
+            Ok(label
+                .parse::<u32>()
+                .unwrap_or_else(|_| self.skip_blocks.unwrap_or(0)))
+        } else {
+            Ok(self.skip_blocks.unwrap_or(0))
+        }
+    }
+
+    /// Set the synced height in the core node by using a label of a fixed address so that
+    /// another client with the same descriptor doesn't rescan the blockchain
+    fn set_node_synced_height(&self, height: u32) -> Result<(), Error> {
+        Ok(self
+            .client
+            .set_label(&self._storage_address, &height.to_string())?)
+    }
+}
+
+impl Blockchain for RpcBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        self.capabilities.clone()
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        let mut scripts_pubkeys = database.iter_script_pubkeys(Some(KeychainKind::External))?;
+        scripts_pubkeys.extend(database.iter_script_pubkeys(Some(KeychainKind::Internal))?);
+        debug!(
+            "importing {} script_pubkeys (some maybe already imported)",
+            scripts_pubkeys.len()
+        );
+        let requests: Vec<_> = scripts_pubkeys
+            .iter()
+            .map(|s| ImportMultiRequest {
+                timestamp: ImportMultiRescanSince::Timestamp(0),
+                script_pubkey: Some(ImportMultiRequestScriptPubkey::Script(&s)),
+                watchonly: Some(true),
+                ..Default::default()
+            })
+            .collect();
+        let options = ImportMultiOptions {
+            rescan: Some(false),
+        };
+        // Note we use import_multi because as of bitcoin core 0.21.0 many descriptors are not supported
+        // https://bitcoindevkit.org/descriptors/#compatibility-matrix
+        //TODO maybe convenient using import_descriptor for compatible descriptor and import_multi as fallback
+        self.client.import_multi(&requests, Some(&options))?;
+
+        let current_height = self.get_height()?;
+
+        // min because block invalidate may cause height to go down
+        let node_synced = self.get_node_synced_height()?.min(current_height);
+
+        //TODO call rescan in chunks (updating node_synced_height) so that in case of
+        // interruption work can be partially recovered
+        debug!(
+            "rescan_blockchain from:{} to:{}",
+            node_synced, current_height
+        );
+        self.client
+            .rescan_blockchain(Some(node_synced as usize), Some(current_height as usize))?;
+        progress_update.update(1.0, None)?;
+
+        self.set_node_synced_height(current_height)?;
+
+        self.sync(stop_gap, database, progress_update)
+    }
+
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        _stop_gap: Option<usize>,
+        db: &mut D,
+        _progress_update: P,
+    ) -> Result<(), Error> {
+        let mut indexes = HashMap::new();
+        for keykind in &[KeychainKind::External, KeychainKind::Internal] {
+            indexes.insert(*keykind, db.get_last_index(*keykind)?.unwrap_or(0));
+        }
+
+        let mut known_txs: HashMap<_, _> = db
+            .iter_txs(true)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let known_utxos: HashSet<_> = db.iter_utxos()?.into_iter().collect();
+
+        //TODO list_since_blocks would be more efficient
+        let current_utxo = self
+            .client
+            .list_unspent(Some(0), None, None, Some(true), None)?;
+        debug!("current_utxo len {}", current_utxo.len());
+
+        //TODO supported up to 1_000 txs, should use since_blocks or do paging
+        let list_txs = self
+            .client
+            .list_transactions(None, Some(1_000), None, Some(true))?;
+        let mut list_txs_ids = HashSet::new();
+
+        for tx_result in list_txs.iter().filter(|t| {
+            // list_txs returns all conflicting tx we want to
+            // filter out replaced tx => unconfirmed and not in the mempool
+            t.info.confirmations > 0 || self.client.get_mempool_entry(&t.info.txid).is_ok()
+        }) {
+            let txid = tx_result.info.txid;
+            list_txs_ids.insert(txid);
+            if let Some(mut known_tx) = known_txs.get_mut(&txid) {
+                let confirmation_time =
+                    ConfirmationTime::new(tx_result.info.blockheight, tx_result.info.blocktime);
+                if confirmation_time != known_tx.confirmation_time {
+                    // reorg may change tx height
+                    debug!(
+                        "updating tx({}) confirmation time to: {:?}",
+                        txid, confirmation_time
+                    );
+                    known_tx.confirmation_time = confirmation_time;
+                    db.set_tx(&known_tx)?;
+                }
+            } else {
+                //TODO check there is already the raw tx in db?
+                let tx_result = self.client.get_transaction(&txid, Some(true))?;
+                let tx: Transaction = deserialize(&tx_result.hex)?;
+                let mut received = 0u64;
+                let mut sent = 0u64;
+                for output in tx.output.iter() {
+                    if let Ok(Some((kind, index))) =
+                        db.get_path_from_script_pubkey(&output.script_pubkey)
+                    {
+                        if index > *indexes.get(&kind).unwrap() {
+                            indexes.insert(kind, index);
+                        }
+                        received += output.value;
+                    }
+                }
+
+                for input in tx.input.iter() {
+                    if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+                        sent += previous_output.value;
+                    }
+                }
+
+                let td = TransactionDetails {
+                    transaction: Some(tx),
+                    txid: tx_result.info.txid,
+                    confirmation_time: ConfirmationTime::new(
+                        tx_result.info.blockheight,
+                        tx_result.info.blocktime,
+                    ),
+                    received,
+                    sent,
+                    fee: tx_result.fee.map(|f| f.as_sat().abs() as u64),
+                    verified: true,
+                };
+                debug!(
+                    "saving tx: {} tx_result.fee:{:?} td.fees:{:?}",
+                    td.txid, tx_result.fee, td.fee
+                );
+                db.set_tx(&td)?;
+            }
+        }
+
+        for known_txid in known_txs.keys() {
+            if !list_txs_ids.contains(known_txid) {
+                debug!("removing tx: {}", known_txid);
+                db.del_tx(known_txid, false)?;
+            }
+        }
+
+        let current_utxos: HashSet<_> = current_utxo
+            .into_iter()
+            .map(|u| {
+                Ok(LocalUtxo {
+                    outpoint: OutPoint::new(u.txid, u.vout),
+                    keychain: db
+                        .get_path_from_script_pubkey(&u.script_pub_key)?
+                        .ok_or(Error::TransactionNotFound)?
+                        .0,
+                    txout: TxOut {
+                        value: u.amount.as_sat(),
+                        script_pubkey: u.script_pub_key,
+                    },
+                })
+            })
+            .collect::<Result<_, Error>>()?;
+
+        let spent: HashSet<_> = known_utxos.difference(&current_utxos).collect();
+        for s in spent {
+            debug!("removing utxo: {:?}", s);
+            db.del_utxo(&s.outpoint)?;
+        }
+        let received: HashSet<_> = current_utxos.difference(&known_utxos).collect();
+        for s in received {
+            debug!("adding utxo: {:?}", s);
+            db.set_utxo(s)?;
+        }
+
+        for (keykind, index) in indexes {
+            debug!("{:?} max {}", keykind, index);
+            db.set_last_index(keykind, index)?;
+        }
+
+        Ok(())
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(Some(self.client.get_raw_transaction(txid, None)?))
+    }
+
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        Ok(self.client.send_raw_transaction(tx).map(|_| ())?)
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(self.client.get_blockchain_info().map(|i| i.blocks as u32)?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        let sat_per_kb = self
+            .client
+            .estimate_smart_fee(target as u16, None)?
+            .fee_rate
+            .ok_or(Error::FeeRateUnavailable)?
+            .as_sat() as f64;
+
+        Ok(FeeRate::from_sat_per_vb((sat_per_kb / 1000f64) as f32))
+    }
+}
+
+impl ConfigurableBlockchain for RpcBlockchain {
+    type Config = RpcConfig;
+
+    /// Returns RpcBlockchain backend creating an RPC client to a specific wallet named as the descriptor's checksum
+    /// if it's the first time it creates the wallet in the node and upon return is granted the wallet is loaded
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let wallet_name = config.wallet_name.clone();
+        let wallet_url = format!("{}/wallet/{}", config.url, &wallet_name);
+        debug!("connecting to {} auth:{:?}", wallet_url, config.auth);
+
+        let client = Client::new(wallet_url, config.auth.clone())?;
+        let loaded_wallets = client.list_wallets()?;
+        if loaded_wallets.contains(&wallet_name) {
+            debug!("wallet already loaded {:?}", wallet_name);
+        } else {
+            let existing_wallets = list_wallet_dir(&client)?;
+            if existing_wallets.contains(&wallet_name) {
+                client.load_wallet(&wallet_name)?;
+                debug!("wallet loaded {:?}", wallet_name);
+            } else {
+                client.create_wallet(&wallet_name, Some(true), None, None, None)?;
+                debug!("wallet created {:?}", wallet_name);
+            }
+        }
+
+        let blockchain_info = client.get_blockchain_info()?;
+        let network = match blockchain_info.chain.as_str() {
+            "main" => Network::Bitcoin,
+            "test" => Network::Testnet,
+            "regtest" => Network::Regtest,
+            "signet" => Network::Signet,
+            _ => return Err(Error::Generic("Invalid network".to_string())),
+        };
+        if network != config.network {
+            return Err(Error::InvalidNetwork {
+                requested: config.network,
+                found: network,
+            });
+        }
+
+        let mut capabilities: HashSet<_> = vec![Capability::FullHistory].into_iter().collect();
+        let rpc_version = client.version()?;
+        if rpc_version >= 210_000 {
+            let info: HashMap<String, Value> = client.call("getindexinfo", &[]).unwrap();
+            if info.contains_key("txindex") {
+                capabilities.insert(Capability::GetAnyTx);
+                capabilities.insert(Capability::AccurateFees);
+            }
+        }
+
+        // this is just a fixed address used only to store a label containing the synced height in the node
+        let mut storage_address =
+            Address::from_str("bc1qst0rewf0wm4kw6qn6kv0e5tc56nkf9yhcxlhqv").unwrap();
+        storage_address.network = network;
+
+        Ok(RpcBlockchain {
+            client,
+            network,
+            capabilities,
+            _storage_address: storage_address,
+            skip_blocks: config.skip_blocks,
+        })
+    }
+}
+
+/// Deterministically generate a unique name given the descriptors defining the wallet
+pub fn wallet_name_from_descriptor<T>(
+    descriptor: T,
+    change_descriptor: Option<T>,
+    network: Network,
+    secp: &SecpCtx,
+) -> Result<String, Error>
+where
+    T: IntoWalletDescriptor,
+{
+    //TODO check descriptors contains only public keys
+    let descriptor = descriptor
+        .into_wallet_descriptor(&secp, network)?
+        .0
+        .to_string();
+    let mut wallet_name = get_checksum(&descriptor[..descriptor.find('#').unwrap()])?;
+    if let Some(change_descriptor) = change_descriptor {
+        let change_descriptor = change_descriptor
+            .into_wallet_descriptor(&secp, network)?
+            .0
+            .to_string();
+        wallet_name.push_str(
+            get_checksum(&change_descriptor[..change_descriptor.find('#').unwrap()])?.as_str(),
+        );
+    }
+
+    Ok(wallet_name)
+}
+
+/// return the wallets available in default wallet directory
+//TODO use bitcoincore_rpc method when PR #179 lands
+fn list_wallet_dir(client: &Client) -> Result<Vec<String>, Error> {
+    #[derive(Deserialize)]
+    struct Name {
+        name: String,
+    }
+    #[derive(Deserialize)]
+    struct CallResult {
+        wallets: Vec<Name>,
+    }
+
+    let result: CallResult = client.call("listwalletdir", &[])?;
+    Ok(result.wallets.into_iter().map(|n| n.name).collect())
+}
+
+#[cfg(feature = "test-blockchains")]
+crate::bdk_blockchain_tests! {
+
+    fn test_instance() -> RpcBlockchain {
+        let url = std::env::var("BDK_RPC_URL").unwrap_or_else(|_| "127.0.0.1:18443".to_string());
+        let url = format!("http://{}", url);
+
+        // TODO same code in `fn get_auth` in testutils, make it public there
+        let auth = match std::env::var("BDK_RPC_AUTH").as_ref().map(String::as_ref) {
+            Ok("USER_PASS") => Auth::UserPass(
+                std::env::var("BDK_RPC_USER").unwrap(),
+                std::env::var("BDK_RPC_PASS").unwrap(),
+            ),
+            _ => Auth::CookieFile(std::path::PathBuf::from(
+                std::env::var("BDK_RPC_COOKIEFILE")
+                    .unwrap_or_else(|_| "/home/user/.bitcoin/regtest/.cookie".to_string()),
+            )),
+        };
+        let config = RpcConfig {
+            url,
+            auth,
+            network: Network::Regtest,
+            wallet_name: format!("client-wallet-test-{:?}", std::time::SystemTime::now() ),
+            skip_blocks: None,
+        };
+        RpcBlockchain::from_config(&config).unwrap()
+    }
+}
+
+#[cfg(feature = "test-rpc")]
+#[cfg(test)]
+mod test {
+    use super::{RpcBlockchain, RpcConfig};
+    use crate::bitcoin::consensus::deserialize;
+    use crate::bitcoin::{Address, Amount, Network, Transaction};
+    use crate::blockchain::rpc::wallet_name_from_descriptor;
+    use crate::blockchain::{noop_progress, Blockchain, Capability, ConfigurableBlockchain};
+    use crate::database::MemoryDatabase;
+    use crate::wallet::AddressIndex;
+    use crate::Wallet;
+    use bitcoin::secp256k1::Secp256k1;
+    use bitcoin::Txid;
+    use bitcoincore_rpc::json::CreateRawTransactionInput;
+    use bitcoincore_rpc::RawTx;
+    use bitcoincore_rpc::{Auth, RpcApi};
+    use bitcoind::BitcoinD;
+    use std::collections::HashMap;
+
+    fn create_rpc(
+        bitcoind: &BitcoinD,
+        desc: &str,
+        network: Network,
+    ) -> Result<RpcBlockchain, crate::Error> {
+        let secp = Secp256k1::new();
+        let wallet_name = wallet_name_from_descriptor(desc, None, network, &secp).unwrap();
+
+        let config = RpcConfig {
+            url: bitcoind.rpc_url(),
+            auth: Auth::CookieFile(bitcoind.config.cookie_file.clone()),
+            network,
+            wallet_name,
+            skip_blocks: None,
+        };
+        RpcBlockchain::from_config(&config)
+    }
+    fn create_bitcoind(args: Vec<String>) -> BitcoinD {
+        let exe = std::env::var("BITCOIND_EXE").unwrap();
+        bitcoind::BitcoinD::with_args(exe, args, false, bitcoind::P2P::No).unwrap()
+    }
+
+    const DESCRIPTOR_PUB: &'static str = "wpkh(tpubD6NzVbkrYhZ4X2yy78HWrr1M9NT8dKeWfzNiQqDdMqqa9UmmGztGGz6TaLFGsLfdft5iu32gxq1T4eMNxExNNWzVCpf9Y6JZi5TnqoC9wJq/*)";
+    const DESCRIPTOR_PRIV: &'static str = "wpkh(tprv8ZgxMBicQKsPdZxBDUcvTSMEaLwCTzTc6gmw8KBKwa3BJzWzec4g6VUbQBHJcutDH6mMEmBeVyN27H1NF3Nu8isZ1Sts4SufWyfLE6Mf1MB/*)";
+
+    #[test]
+    fn test_rpc_wallet_setup() {
+        env_logger::try_init().unwrap();
+        let bitcoind = create_bitcoind(vec![]);
+        let node_address = bitcoind.client.get_new_address(None, None).unwrap();
+        let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
+        let db = MemoryDatabase::new();
+        let wallet = Wallet::new(DESCRIPTOR_PRIV, None, Network::Regtest, db, blockchain).unwrap();
+
+        wallet.sync(noop_progress(), None).unwrap();
+        generate(&bitcoind, 101);
+        wallet.sync(noop_progress(), None).unwrap();
+        let address = wallet.get_address(AddressIndex::New).unwrap();
+        let expected_address = "bcrt1q8dyvgt4vhr8ald4xuwewcxhdjha9a5k78wxm5t";
+        assert_eq!(expected_address, address.to_string());
+        send_to_address(&bitcoind, &address, 100_000);
+        wallet.sync(noop_progress(), None).unwrap();
+        assert_eq!(wallet.get_balance().unwrap(), 100_000);
+
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(node_address.script_pubkey(), 50_000);
+        let (mut psbt, details) = builder.finish().unwrap();
+        let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+        assert!(finalized, "Cannot finalize transaction");
+        let tx = psbt.extract_tx();
+        wallet.broadcast(tx).unwrap();
+        wallet.sync(noop_progress(), None).unwrap();
+        assert_eq!(
+            wallet.get_balance().unwrap(),
+            100_000 - 50_000 - details.fee.unwrap_or(0)
+        );
+        drop(wallet);
+
+        // test skip_blocks
+        generate(&bitcoind, 5);
+        let config = RpcConfig {
+            url: bitcoind.rpc_url(),
+            auth: Auth::CookieFile(bitcoind.config.cookie_file.clone()),
+            network: Network::Regtest,
+            wallet_name: "another-name".to_string(),
+            skip_blocks: Some(103),
+        };
+        let blockchain_skip = RpcBlockchain::from_config(&config).unwrap();
+        let db = MemoryDatabase::new();
+        let wallet_skip =
+            Wallet::new(DESCRIPTOR_PRIV, None, Network::Regtest, db, blockchain_skip).unwrap();
+        wallet_skip.sync(noop_progress(), None).unwrap();
+        send_to_address(&bitcoind, &address, 100_000);
+        wallet_skip.sync(noop_progress(), None).unwrap();
+        assert_eq!(wallet_skip.get_balance().unwrap(), 100_000);
+    }
+
+    #[test]
+    fn test_rpc_from_config() {
+        let bitcoind = create_bitcoind(vec![]);
+        let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest);
+        assert!(blockchain.is_ok());
+        let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Testnet);
+        assert!(blockchain.is_err(), "wrong network doesn't error");
+    }
+
+    #[test]
+    fn test_rpc_capabilities_get_tx() {
+        let bitcoind = create_bitcoind(vec![]);
+        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
+        let capabilities = rpc.get_capabilities();
+        assert!(capabilities.contains(&Capability::FullHistory) && capabilities.len() == 1);
+        let bitcoind_indexed = create_bitcoind(vec!["-txindex".to_string()]);
+        let rpc_indexed = create_rpc(&bitcoind_indexed, DESCRIPTOR_PUB, Network::Regtest).unwrap();
+        assert_eq!(rpc_indexed.get_capabilities().len(), 3);
+        let address = generate(&bitcoind_indexed, 101);
+        let txid = send_to_address(&bitcoind_indexed, &address, 100_000);
+        assert!(rpc_indexed.get_tx(&txid).unwrap().is_some());
+        assert!(rpc.get_tx(&txid).is_err());
+    }
+
+    #[test]
+    fn test_rpc_estimate_fee_get_height() {
+        let bitcoind = create_bitcoind(vec![]);
+        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
+        let result = rpc.estimate_fee(2);
+        assert!(result.is_err());
+        let address = generate(&bitcoind, 100);
+        // create enough tx so that core give some fee estimation
+        for _ in 0..15 {
+            let _ = bitcoind.client.generate_to_address(1, &address).unwrap();
+            for _ in 0..2 {
+                send_to_address(&bitcoind, &address, 100_000);
+            }
+        }
+        let result = rpc.estimate_fee(2);
+        assert!(result.is_ok());
+        assert_eq!(rpc.get_height().unwrap(), 115);
+    }
+
+    #[test]
+    fn test_rpc_node_synced_height() {
+        let bitcoind = create_bitcoind(vec![]);
+        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
+        let synced_height = rpc.get_node_synced_height().unwrap();
+
+        assert_eq!(synced_height, 0);
+        rpc.set_node_synced_height(1).unwrap();
+
+        let synced_height = rpc.get_node_synced_height().unwrap();
+        assert_eq!(synced_height, 1);
+    }
+
+    #[test]
+    fn test_rpc_broadcast() {
+        let bitcoind = create_bitcoind(vec![]);
+        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
+        let address = generate(&bitcoind, 101);
+        let utxo = bitcoind
+            .client
+            .list_unspent(None, None, None, None, None)
+            .unwrap();
+        let input = CreateRawTransactionInput {
+            txid: utxo[0].txid,
+            vout: utxo[0].vout,
+            sequence: None,
+        };
+
+        let out: HashMap<_, _> = vec![(
+            address.to_string(),
+            utxo[0].amount - Amount::from_sat(100_000),
+        )]
+        .into_iter()
+        .collect();
+        let tx = bitcoind
+            .client
+            .create_raw_transaction(&[input], &out, None, None)
+            .unwrap();
+        let signed_tx = bitcoind
+            .client
+            .sign_raw_transaction_with_wallet(tx.raw_hex(), None, None)
+            .unwrap();
+        let parsed_tx: Transaction = deserialize(&signed_tx.hex).unwrap();
+        rpc.broadcast(&parsed_tx).unwrap();
+        assert!(bitcoind
+            .client
+            .get_raw_mempool()
+            .unwrap()
+            .contains(&tx.txid()));
+    }
+
+    #[test]
+    fn test_rpc_wallet_name() {
+        let secp = Secp256k1::new();
+        let name =
+            wallet_name_from_descriptor(DESCRIPTOR_PUB, None, Network::Regtest, &secp).unwrap();
+        assert_eq!("tmg7aqay", name);
+    }
+
+    fn generate(bitcoind: &BitcoinD, blocks: u64) -> Address {
+        let address = bitcoind.client.get_new_address(None, None).unwrap();
+        bitcoind
+            .client
+            .generate_to_address(blocks, &address)
+            .unwrap();
+        address
+    }
+
+    fn send_to_address(bitcoind: &BitcoinD, address: &Address, amount: u64) -> Txid {
+        bitcoind
+            .client
+            .send_to_address(
+                &address,
+                Amount::from_sat(amount),
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+            )
+            .unwrap()
+    }
+}

src/blockchain/utils.rs

@@ -1,358 +1,389 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
-use std::cmp;
-use std::collections::{HashSet, VecDeque};
-use std::convert::TryFrom;
+use std::collections::{HashMap, HashSet};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
+use rand::seq::SliceRandom;
+use rand::thread_rng;
 
-use bitcoin::{Address, Network, OutPoint, Script, Transaction, Txid};
+use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 
 use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{ScriptType, TransactionDetails, UTXO};
+use crate::types::{ConfirmationTime, KeychainKind, LocalUtxo, TransactionDetails};
+use crate::wallet::time::Instant;
 use crate::wallet::utils::ChunksIterator;
 
 #[derive(Debug)]
-pub struct ELSGetHistoryRes {
+pub struct ElsGetHistoryRes {
     pub height: i32,
     pub tx_hash: Txid,
 }
 
-#[derive(Debug)]
-pub struct ELSListUnspentRes {
-    pub height: usize,
-    pub tx_hash: Txid,
-    pub tx_pos: usize,
-}
-
 /// Implements the synchronization logic for an Electrum-like client.
 #[maybe_async]
 pub trait ElectrumLikeSync {
-    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
         &self,
         scripts: I,
-    ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error>;
+    ) -> Result<Vec<Vec<ElsGetHistoryRes>>, Error>;
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid> + Clone>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error>;
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error>;
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error>;
+    fn els_batch_block_header<I: IntoIterator<Item = u32> + Clone>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error>;
 
     // Provided methods down here...
 
     fn electrum_like_setup<D: BatchDatabase, P: Progress>(
         &self,
         stop_gap: Option<usize>,
-        database: &mut D,
+        db: &mut D,
         _progress_update: P,
     ) -> Result<(), Error> {
         // TODO: progress
+        let start = Instant::new();
+        debug!("start setup");
 
         let stop_gap = stop_gap.unwrap_or(20);
-        let batch_query_size = 20;
+        let chunk_size = stop_gap;
 
-        // check unconfirmed tx, delete so they are retrieved later
-        let mut del_batch = database.begin_batch();
-        for tx in database.iter_txs(false)? {
-            if tx.height.is_none() {
-                del_batch.del_tx(&tx.txid, false)?;
-            }
-        }
-        database.commit_batch(del_batch)?;
+        let mut history_txs_id = HashSet::new();
+        let mut txid_height = HashMap::new();
+        let mut max_indexes = HashMap::new();
 
-        // maximum derivation index for a change address that we've seen during sync
-        let mut change_max_deriv = None;
+        let mut wallet_chains = vec![KeychainKind::Internal, KeychainKind::External];
+        // shuffling improve privacy, the server doesn't know my first request is from my internal or external addresses
+        wallet_chains.shuffle(&mut thread_rng());
+        // download history of our internal and external script_pubkeys
+        for keychain in wallet_chains.iter() {
+            let script_iter = db.iter_script_pubkeys(Some(*keychain))?.into_iter();
 
-        let mut already_checked: HashSet<Script> = HashSet::new();
-        let mut to_check_later = VecDeque::with_capacity(batch_query_size);
-
-        // insert the first chunk
-        let mut iter_scriptpubkeys = database
-            .iter_script_pubkeys(Some(ScriptType::External))?
-            .into_iter();
-        let chunk: Vec<Script> = iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-        for item in chunk.into_iter().rev() {
-            to_check_later.push_front(item);
-        }
-
-        let mut iterating_external = true;
-        let mut index = 0;
-        let mut last_found = None;
-        while !to_check_later.is_empty() {
-            trace!("to_check_later size {}", to_check_later.len());
-
-            let until = cmp::min(to_check_later.len(), batch_query_size);
-            let chunk: Vec<Script> = to_check_later.drain(..until).collect();
-            let call_result = maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
-
-            for (script, history) in chunk.into_iter().zip(call_result.into_iter()) {
-                trace!("received history for {:?}, size {}", script, history.len());
-
-                if !history.is_empty() {
-                    last_found = Some(index);
-
-                    let mut check_later_scripts = maybe_await!(self.check_history(
-                        database,
-                        script,
-                        history,
-                        &mut change_max_deriv
-                    ))?
-                    .into_iter()
-                    .filter(|x| already_checked.insert(x.clone()))
-                    .collect();
-                    to_check_later.append(&mut check_later_scripts);
+            for (i, chunk) in ChunksIterator::new(script_iter, stop_gap).enumerate() {
+                // TODO if i == last, should create another chunk of addresses in db
+                let call_result: Vec<Vec<ElsGetHistoryRes>> =
+                    maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
+                let max_index = call_result
+                    .iter()
+                    .enumerate()
+                    .filter_map(|(i, v)| v.first().map(|_| i as u32))
+                    .max();
+                if let Some(max) = max_index {
+                    max_indexes.insert(keychain, max + (i * chunk_size) as u32);
+                }
+                let flattened: Vec<ElsGetHistoryRes> = call_result.into_iter().flatten().collect();
+                debug!("#{} of {:?} results:{}", i, keychain, flattened.len());
+                if flattened.is_empty() {
+                    // Didn't find anything in the last `stop_gap` script_pubkeys, breaking
+                    break;
                 }
 
-                index += 1;
-            }
-
-            match iterating_external {
-                true if index - last_found.unwrap_or(0) >= stop_gap => iterating_external = false,
-                true => {
-                    trace!("pushing one more batch from `iter_scriptpubkeys`. index = {}, last_found = {:?}, stop_gap = {}", index, last_found, stop_gap);
-
-                    let chunk: Vec<Script> =
-                        iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-                    for item in chunk.into_iter().rev() {
-                        to_check_later.push_front(item);
+                for el in flattened {
+                    // el.height = -1 means unconfirmed with unconfirmed parents
+                    // el.height =  0 means unconfirmed with confirmed parents
+                    // but we treat those tx the same
+                    if el.height <= 0 {
+                        txid_height.insert(el.tx_hash, None);
+                    } else {
+                        txid_height.insert(el.tx_hash, Some(el.height as u32));
                     }
-                }
-                _ => {}
-            }
-        }
-
-        // check utxo
-        // TODO: try to minimize network requests and re-use scripts if possible
-        let mut batch = database.begin_batch();
-        for chunk in ChunksIterator::new(database.iter_utxos()?.into_iter(), batch_query_size) {
-            let scripts: Vec<_> = chunk.iter().map(|u| &u.txout.script_pubkey).collect();
-            let call_result = maybe_await!(self.els_batch_script_list_unspent(scripts))?;
-
-            // check which utxos are actually still unspent
-            for (utxo, list_unspent) in chunk.into_iter().zip(call_result.iter()) {
-                debug!(
-                    "outpoint {:?} is unspent for me, list unspent is {:?}",
-                    utxo.outpoint, list_unspent
-                );
-
-                let mut spent = true;
-                for unspent in list_unspent {
-                    let res_outpoint = OutPoint::new(unspent.tx_hash, unspent.tx_pos as u32);
-                    if utxo.outpoint == res_outpoint {
-                        spent = false;
-                        break;
-                    }
-                }
-                if spent {
-                    info!("{} not anymore unspent, removing", utxo.outpoint);
-                    batch.del_utxo(&utxo.outpoint)?;
+                    history_txs_id.insert(el.tx_hash);
                 }
             }
         }
 
-        let current_ext = database.get_last_index(ScriptType::External)?.unwrap_or(0);
-        let first_ext_new = last_found.map(|x| x + 1).unwrap_or(0) as u32;
-        if first_ext_new > current_ext {
-            info!("Setting external index to {}", first_ext_new);
-            database.set_last_index(ScriptType::External, first_ext_new)?;
+        // saving max indexes
+        info!("max indexes are: {:?}", max_indexes);
+        for keychain in wallet_chains.iter() {
+            if let Some(index) = max_indexes.get(keychain) {
+                db.set_last_index(*keychain, *index)?;
+            }
         }
 
-        let current_int = database.get_last_index(ScriptType::Internal)?.unwrap_or(0);
-        let first_int_new = change_max_deriv.map(|x| x + 1).unwrap_or(0);
-        if first_int_new > current_int {
-            info!("Setting internal index to {}", first_int_new);
-            database.set_last_index(ScriptType::Internal, first_int_new)?;
+        // get db status
+        let txs_details_in_db: HashMap<Txid, TransactionDetails> = db
+            .iter_txs(false)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let txs_raw_in_db: HashMap<Txid, Transaction> = db
+            .iter_raw_txs()?
+            .into_iter()
+            .map(|tx| (tx.txid(), tx))
+            .collect();
+        let utxos_deps = utxos_deps(db, &txs_raw_in_db)?;
+
+        // download new txs and headers
+        let new_txs = maybe_await!(self.download_and_save_needed_raw_txs(
+            &history_txs_id,
+            &txs_raw_in_db,
+            chunk_size,
+            db
+        ))?;
+        let new_timestamps = maybe_await!(self.download_needed_headers(
+            &txid_height,
+            &txs_details_in_db,
+            chunk_size
+        ))?;
+
+        let mut batch = db.begin_batch();
+
+        // save any tx details not in db but in history_txs_id or with different height/timestamp
+        for txid in history_txs_id.iter() {
+            let height = txid_height.get(txid).cloned().flatten();
+            let timestamp = new_timestamps.get(txid).cloned();
+            if let Some(tx_details) = txs_details_in_db.get(txid) {
+                // check if tx height matches, otherwise updates it. timestamp is not in the if clause
+                // because we are not asking headers for confirmed tx we know about
+                if tx_details.confirmation_time.as_ref().map(|c| c.height) != height {
+                    let confirmation_time = ConfirmationTime::new(height, timestamp);
+                    let mut new_tx_details = tx_details.clone();
+                    new_tx_details.confirmation_time = confirmation_time;
+                    batch.set_tx(&new_tx_details)?;
+                }
+            } else {
+                save_transaction_details_and_utxos(
+                    txid,
+                    db,
+                    timestamp,
+                    height,
+                    &mut batch,
+                    &utxos_deps,
+                )?;
+            }
         }
 
-        database.commit_batch(batch)?;
+        // remove any tx details in db but not in history_txs_id
+        for txid in txs_details_in_db.keys() {
+            if !history_txs_id.contains(txid) {
+                batch.del_tx(txid, false)?;
+            }
+        }
+
+        // remove any spent utxo
+        for new_tx in new_txs.iter() {
+            for input in new_tx.input.iter() {
+                batch.del_utxo(&input.previous_output)?;
+            }
+        }
+
+        db.commit_batch(batch)?;
+        info!("finish setup, elapsed {:?}ms", start.elapsed().as_millis());
 
         Ok(())
     }
 
-    fn check_tx_and_descendant<D: BatchDatabase>(
+    /// download txs identified by `history_txs_id` and theirs previous outputs if not already present in db
+    fn download_and_save_needed_raw_txs<D: BatchDatabase>(
         &self,
-        database: &mut D,
-        txid: &Txid,
-        height: Option<u32>,
-        cur_script: &Script,
-        change_max_deriv: &mut Option<u32>,
-    ) -> Result<Vec<Script>, Error> {
-        debug!(
-            "check_tx_and_descendant of {}, height: {:?}, script: {}",
-            txid, height, cur_script
-        );
-        let mut updates = database.begin_batch();
-        let tx = match database.get_tx(&txid, true)? {
-            Some(mut saved_tx) => {
-                // update the height if it's different (in case of reorg)
-                if saved_tx.height != height {
-                    info!(
-                        "updating height from {:?} to {:?} for tx {}",
-                        saved_tx.height, height, txid
-                    );
-                    saved_tx.height = height;
-                    updates.set_tx(&saved_tx)?;
-                }
-
-                debug!("already have {} in db, returning the cached version", txid);
-
-                // unwrap since we explicitly ask for the raw_tx, if it's not present something
-                // went wrong
-                saved_tx.transaction.unwrap()
-            }
-            None => {
-                let fetched_tx = maybe_await!(self.els_transaction_get(&txid))?;
-                database.set_raw_tx(&fetched_tx)?;
-
-                fetched_tx
-            }
-        };
-
-        let mut incoming: u64 = 0;
-        let mut outgoing: u64 = 0;
-
-        let mut inputs_sum: u64 = 0;
-        let mut outputs_sum: u64 = 0;
-
-        // look for our own inputs
-        for (i, input) in tx.input.iter().enumerate() {
-            // the fact that we visit addresses in a BFS fashion starting from the external addresses
-            // should ensure that this query is always consistent (i.e. when we get to call this all
-            // the transactions at a lower depth have already been indexed, so if an outpoint is ours
-            // we are guaranteed to have it in the db).
-            if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
-                inputs_sum += previous_output.value;
-
-                if database.is_mine(&previous_output.script_pubkey)? {
-                    outgoing += previous_output.value;
-
-                    debug!("{} input #{} is mine, removing from utxo", txid, i);
-                    updates.del_utxo(&input.previous_output)?;
-                }
-            } else {
-                // The input is not ours, but we still need to count it for the fees. so fetch the
-                // tx (from the database or from network) and check it
-                let tx = match database.get_tx(&input.previous_output.txid, true)? {
-                    Some(saved_tx) => saved_tx.transaction.unwrap(),
-                    None => {
-                        let fetched_tx =
-                            maybe_await!(self.els_transaction_get(&input.previous_output.txid))?;
-                        database.set_raw_tx(&fetched_tx)?;
-
-                        fetched_tx
-                    }
-                };
-
-                inputs_sum += tx.output[input.previous_output.vout as usize].value;
-            }
-        }
-
-        let mut to_check_later = vec![];
-        for (i, output) in tx.output.iter().enumerate() {
-            // to compute the fees later
-            outputs_sum += output.value;
-
-            // this output is ours, we have a path to derive it
-            if let Some((script_type, child)) =
-                database.get_path_from_script_pubkey(&output.script_pubkey)?
-            {
-                debug!("{} output #{} is mine, adding utxo", txid, i);
-                updates.set_utxo(&UTXO {
-                    outpoint: OutPoint::new(tx.txid(), i as u32),
-                    txout: output.clone(),
-                    is_internal: script_type.is_internal(),
-                })?;
-                incoming += output.value;
-
-                if output.script_pubkey != *cur_script {
-                    debug!("{} output #{} script {} was not current script, adding script to be checked later", txid, i, output.script_pubkey);
-                    to_check_later.push(output.script_pubkey.clone())
-                }
-
-                // derive as many change addrs as external addresses that we've seen
-                if script_type == ScriptType::Internal
-                    && (change_max_deriv.is_none() || child > change_max_deriv.unwrap_or(0))
-                {
-                    *change_max_deriv = Some(child);
+        history_txs_id: &HashSet<Txid>,
+        txs_raw_in_db: &HashMap<Txid, Transaction>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        let txids_raw_in_db: HashSet<Txid> = txs_raw_in_db.keys().cloned().collect();
+        let txids_to_download: Vec<&Txid> = history_txs_id.difference(&txids_raw_in_db).collect();
+        if !txids_to_download.is_empty() {
+            info!("got {} txs to download", txids_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                txids_to_download,
+                chunk_size,
+                db,
+            ))?);
+            let mut prev_txids = HashSet::new();
+            let mut txids_downloaded = HashSet::new();
+            for tx in txs_downloaded.iter() {
+                txids_downloaded.insert(tx.txid());
+                // add every previous input tx, but skip coinbase
+                for input in tx.input.iter().filter(|i| !i.previous_output.is_null()) {
+                    prev_txids.insert(input.previous_output.txid);
                 }
             }
-        }
-
-        let tx = TransactionDetails {
-            txid: tx.txid(),
-            transaction: Some(tx),
-            received: incoming,
-            sent: outgoing,
-            height,
-            timestamp: 0,
-            fees: inputs_sum - outputs_sum,
-        };
-        info!("Saving tx {}", txid);
-        updates.set_tx(&tx)?;
-
-        database.commit_batch(updates)?;
-
-        Ok(to_check_later)
-    }
-
-    fn check_history<D: BatchDatabase>(
-        &self,
-        database: &mut D,
-        script_pubkey: Script,
-        txs: Vec<ELSGetHistoryRes>,
-        change_max_deriv: &mut Option<u32>,
-    ) -> Result<Vec<Script>, Error> {
-        let mut to_check_later = Vec::new();
-
-        debug!(
-            "history of {} script {} has {} tx",
-            Address::from_script(&script_pubkey, Network::Testnet).unwrap(),
-            script_pubkey,
-            txs.len()
-        );
-
-        for tx in txs {
-            let height: Option<u32> = match tx.height {
-                0 | -1 => None,
-                x => u32::try_from(x).ok(),
-            };
-
-            to_check_later.extend_from_slice(&maybe_await!(self.check_tx_and_descendant(
-                database,
-                &tx.tx_hash,
-                height,
-                &script_pubkey,
-                change_max_deriv,
+            let already_present: HashSet<Txid> =
+                txids_downloaded.union(&txids_raw_in_db).cloned().collect();
+            let prev_txs_to_download: Vec<&Txid> =
+                prev_txids.difference(&already_present).collect();
+            info!("{} previous txs to download", prev_txs_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                prev_txs_to_download,
+                chunk_size,
+                db,
             ))?);
         }
 
-        Ok(to_check_later)
+        Ok(txs_downloaded)
+    }
+
+    /// download headers at heights in `txid_height` if tx details not already present, returns a map Txid -> timestamp
+    fn download_needed_headers(
+        &self,
+        txid_height: &HashMap<Txid, Option<u32>>,
+        txs_details_in_db: &HashMap<Txid, TransactionDetails>,
+        chunk_size: usize,
+    ) -> Result<HashMap<Txid, u64>, Error> {
+        let mut txid_timestamp = HashMap::new();
+        let txid_in_db_with_conf: HashSet<_> = txs_details_in_db
+            .values()
+            .filter_map(|details| details.confirmation_time.as_ref().map(|_| details.txid))
+            .collect();
+        let needed_txid_height: HashMap<&Txid, u32> = txid_height
+            .iter()
+            .filter(|(t, _)| !txid_in_db_with_conf.contains(*t))
+            .filter_map(|(t, o)| o.map(|h| (t, h)))
+            .collect();
+        let needed_heights: HashSet<u32> = needed_txid_height.values().cloned().collect();
+        if !needed_heights.is_empty() {
+            info!("{} headers to download for timestamp", needed_heights.len());
+            let mut height_timestamp: HashMap<u32, u64> = HashMap::new();
+            for chunk in ChunksIterator::new(needed_heights.into_iter(), chunk_size) {
+                let call_result: Vec<BlockHeader> =
+                    maybe_await!(self.els_batch_block_header(chunk.clone()))?;
+                height_timestamp.extend(
+                    chunk
+                        .into_iter()
+                        .zip(call_result.iter().map(|h| h.time as u64)),
+                );
+            }
+            for (txid, height) in needed_txid_height {
+                let timestamp = height_timestamp
+                    .get(&height)
+                    .ok_or_else(|| Error::Generic("timestamp missing".to_string()))?;
+                txid_timestamp.insert(*txid, *timestamp);
+            }
+        }
+
+        Ok(txid_timestamp)
+    }
+
+    fn download_and_save_in_chunks<D: BatchDatabase>(
+        &self,
+        to_download: Vec<&Txid>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        for chunk in ChunksIterator::new(to_download.into_iter(), chunk_size) {
+            let call_result: Vec<Transaction> =
+                maybe_await!(self.els_batch_transaction_get(chunk))?;
+            let mut batch = db.begin_batch();
+            for new_tx in call_result.iter() {
+                batch.set_raw_tx(new_tx)?;
+            }
+            db.commit_batch(batch)?;
+            txs_downloaded.extend(call_result);
+        }
+
+        Ok(txs_downloaded)
     }
 }
+
+fn save_transaction_details_and_utxos<D: BatchDatabase>(
+    txid: &Txid,
+    db: &mut D,
+    timestamp: Option<u64>,
+    height: Option<u32>,
+    updates: &mut dyn BatchOperations,
+    utxo_deps: &HashMap<OutPoint, OutPoint>,
+) -> Result<(), Error> {
+    let tx = db.get_raw_tx(txid)?.ok_or(Error::TransactionNotFound)?;
+
+    let mut incoming: u64 = 0;
+    let mut outgoing: u64 = 0;
+
+    let mut inputs_sum: u64 = 0;
+    let mut outputs_sum: u64 = 0;
+
+    // look for our own inputs
+    for input in tx.input.iter() {
+        // skip coinbase inputs
+        if input.previous_output.is_null() {
+            continue;
+        }
+
+        // We already downloaded all previous output txs in the previous step
+        if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+            inputs_sum += previous_output.value;
+
+            if db.is_mine(&previous_output.script_pubkey)? {
+                outgoing += previous_output.value;
+            }
+        } else {
+            // The input is not ours, but we still need to count it for the fees
+            let tx = db
+                .get_raw_tx(&input.previous_output.txid)?
+                .ok_or(Error::TransactionNotFound)?;
+            inputs_sum += tx.output[input.previous_output.vout as usize].value;
+        }
+
+        // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
+        if let Some(outpoint) = utxo_deps.get(&input.previous_output) {
+            updates.del_utxo(outpoint)?;
+        }
+    }
+
+    for (i, output) in tx.output.iter().enumerate() {
+        // to compute the fees later
+        outputs_sum += output.value;
+
+        // this output is ours, we have a path to derive it
+        if let Some((keychain, _child)) = db.get_path_from_script_pubkey(&output.script_pubkey)? {
+            debug!("{} output #{} is mine, adding utxo", txid, i);
+            updates.set_utxo(&LocalUtxo {
+                outpoint: OutPoint::new(tx.txid(), i as u32),
+                txout: output.clone(),
+                keychain,
+            })?;
+
+            incoming += output.value;
+        }
+    }
+
+    let tx_details = TransactionDetails {
+        txid: tx.txid(),
+        transaction: Some(tx),
+        received: incoming,
+        sent: outgoing,
+        confirmation_time: ConfirmationTime::new(height, timestamp),
+        fee: Some(inputs_sum.saturating_sub(outputs_sum)), /* if the tx is a coinbase, fees would be negative */
+        verified: height.is_some(),
+    };
+    updates.set_tx(&tx_details)?;
+
+    Ok(())
+}
+
+/// returns utxo dependency as the inputs needed for the utxo to exist
+/// `tx_raw_in_db` must contains utxo's generating txs or errors witt [crate::Error::TransactionNotFound]
+fn utxos_deps<D: BatchDatabase>(
+    db: &mut D,
+    tx_raw_in_db: &HashMap<Txid, Transaction>,
+) -> Result<HashMap<OutPoint, OutPoint>, Error> {
+    let utxos = db.iter_utxos()?;
+    let mut utxos_deps = HashMap::new();
+    for utxo in utxos {
+        let from_tx = tx_raw_in_db
+            .get(&utxo.outpoint.txid)
+            .ok_or(Error::TransactionNotFound)?;
+        for input in from_tx.input.iter() {
+            utxos_deps.insert(input.previous_output, utxo.outpoint);
+        }
+    }
+    Ok(utxos_deps)
+}

src/cli.rs

@@ -1,544 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-use std::collections::BTreeMap;
-use std::str::FromStr;
-
-use clap::{App, Arg, ArgMatches, SubCommand};
-
-#[allow(unused_imports)]
-use log::{debug, error, info, trace, LevelFilter};
-
-use bitcoin::consensus::encode::{deserialize, serialize, serialize_hex};
-use bitcoin::hashes::hex::FromHex;
-use bitcoin::util::psbt::PartiallySignedTransaction;
-use bitcoin::{Address, OutPoint, Script, Txid};
-
-use crate::blockchain::log_progress;
-use crate::error::Error;
-use crate::types::ScriptType;
-use crate::{FeeRate, TxBuilder, Wallet};
-
-fn parse_recipient(s: &str) -> Result<(Script, u64), String> {
-    let parts: Vec<_> = s.split(":").collect();
-    if parts.len() != 2 {
-        return Err("Invalid format".to_string());
-    }
-
-    let addr = Address::from_str(&parts[0]);
-    if let Err(e) = addr {
-        return Err(format!("{:?}", e));
-    }
-    let val = u64::from_str(&parts[1]);
-    if let Err(e) = val {
-        return Err(format!("{:?}", e));
-    }
-
-    Ok((addr.unwrap().script_pubkey(), val.unwrap()))
-}
-
-fn parse_outpoint(s: &str) -> Result<OutPoint, String> {
-    OutPoint::from_str(s).map_err(|e| format!("{:?}", e))
-}
-
-fn recipient_validator(s: String) -> Result<(), String> {
-    parse_recipient(&s).map(|_| ())
-}
-
-fn outpoint_validator(s: String) -> Result<(), String> {
-    parse_outpoint(&s).map(|_| ())
-}
-
-pub fn make_cli_subcommands<'a, 'b>() -> App<'a, 'b> {
-    App::new("Magical Bitcoin Wallet")
-        .version(option_env!("CARGO_PKG_VERSION").unwrap_or("unknown"))
-        .author(option_env!("CARGO_PKG_AUTHORS").unwrap_or(""))
-        .about("A modern, lightweight, descriptor-based wallet")
-        .subcommand(
-            SubCommand::with_name("get_new_address").about("Generates a new external address"),
-        )
-        .subcommand(SubCommand::with_name("sync").about("Syncs with the chosen Electrum server"))
-        .subcommand(
-            SubCommand::with_name("list_unspent").about("Lists the available spendable UTXOs"),
-        )
-        .subcommand(
-            SubCommand::with_name("list_transactions").about("Lists all the incoming and outgoing transactions of the wallet"),
-        )
-        .subcommand(
-            SubCommand::with_name("get_balance").about("Returns the current wallet balance"),
-        )
-        .subcommand(
-            SubCommand::with_name("create_tx")
-                .about("Creates a new unsigned tranasaction")
-                .arg(
-                    Arg::with_name("to")
-                        .long("to")
-                        .value_name("ADDRESS:SAT")
-                        .help("Adds a recipient to the transaction")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .required(true)
-                        .multiple(true)
-                        .validator(recipient_validator),
-                )
-                .arg(
-                    Arg::with_name("send_all")
-                        .short("all")
-                        .long("send_all")
-                        .help("Sends all the funds (or all the selected utxos). Requires only one recipients of value 0"),
-                )
-                .arg(
-                    Arg::with_name("enable_rbf")
-                        .short("rbf")
-                        .long("enable_rbf")
-                        .help("Enables Replace-By-Fee (BIP125)"),
-                )
-                .arg(
-                    Arg::with_name("utxos")
-                        .long("utxos")
-                        .value_name("TXID:VOUT")
-                        .help("Selects which utxos *must* be spent")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .multiple(true)
-                        .validator(outpoint_validator),
-                )
-                .arg(
-                    Arg::with_name("unspendable")
-                        .long("unspendable")
-                        .value_name("TXID:VOUT")
-                        .help("Marks an utxo as unspendable")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .multiple(true)
-                        .validator(outpoint_validator),
-                )
-                .arg(
-                    Arg::with_name("fee_rate")
-                        .short("fee")
-                        .long("fee_rate")
-                        .value_name("SATS_VBYTE")
-                        .help("Fee rate to use in sat/vbyte")
-                        .takes_value(true),
-                )
-                .arg(
-                    Arg::with_name("policy")
-                        .long("policy")
-                        .value_name("POLICY")
-                        .help("Selects which policy should be used to satisfy the descriptor")
-                        .takes_value(true)
-                        .number_of_values(1),
-                ),
-        )
-        .subcommand(
-            SubCommand::with_name("bump_fee")
-                .about("Bumps the fees of an RBF transaction")
-                .arg(
-                    Arg::with_name("txid")
-                        .required(true)
-                        .takes_value(true)
-                        .short("txid")
-                        .long("txid")
-                        .help("TXID of the transaction to update"),
-                )
-                .arg(
-                    Arg::with_name("send_all")
-                        .short("all")
-                        .long("send_all")
-                        .help("Allows the wallet to reduce the amount of the only output in order to increase fees. This is generally the expected behavior for transactions originally created with `send_all`"),
-                )
-                .arg(
-                    Arg::with_name("utxos")
-                        .long("utxos")
-                        .value_name("TXID:VOUT")
-                        .help("Selects which utxos *must* be added to the tx. Unconfirmed utxos cannot be used")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .multiple(true)
-                        .validator(outpoint_validator),
-                )
-                .arg(
-                    Arg::with_name("unspendable")
-                        .long("unspendable")
-                        .value_name("TXID:VOUT")
-                        .help("Marks an utxo as unspendable, in case more inputs are needed to cover the extra fees")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .multiple(true)
-                        .validator(outpoint_validator),
-                )
-                .arg(
-                    Arg::with_name("fee_rate")
-                        .required(true)
-                        .short("fee")
-                        .long("fee_rate")
-                        .value_name("SATS_VBYTE")
-                        .help("The new targeted fee rate in sat/vbyte")
-                        .takes_value(true),
-                ),
-        )
-        .subcommand(
-            SubCommand::with_name("policies")
-                .about("Returns the available spending policies for the descriptor")
-            )
-        .subcommand(
-            SubCommand::with_name("public_descriptor")
-                .about("Returns the public version of the wallet's descriptor(s)")
-            )
-        .subcommand(
-            SubCommand::with_name("sign")
-                .about("Signs and tries to finalize a PSBT")
-                .arg(
-                    Arg::with_name("psbt")
-                        .long("psbt")
-                        .value_name("BASE64_PSBT")
-                        .help("Sets the PSBT to sign")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("assume_height")
-                        .long("assume_height")
-                        .value_name("HEIGHT")
-                        .help("Assume the blockchain has reached a specific height. This affects the transaction finalization, if there are timelocks in the descriptor")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .required(false),
-                ))
-        .subcommand(
-            SubCommand::with_name("broadcast")
-                .about("Broadcasts a transaction to the network. Takes either a raw transaction or a PSBT to extract")
-                .arg(
-                    Arg::with_name("psbt")
-                        .long("psbt")
-                        .value_name("BASE64_PSBT")
-                        .help("Sets the PSBT to extract and broadcast")
-                        .takes_value(true)
-                        .required_unless("tx")
-                        .number_of_values(1))
-                .arg(
-                    Arg::with_name("tx")
-                        .long("tx")
-                        .value_name("RAWTX")
-                        .help("Sets the raw transaction to broadcast")
-                        .takes_value(true)
-                        .required_unless("psbt")
-                        .number_of_values(1))
-                )
-        .subcommand(
-            SubCommand::with_name("extract_psbt")
-                .about("Extracts a raw transaction from a PSBT")
-                .arg(
-                    Arg::with_name("psbt")
-                        .long("psbt")
-                        .value_name("BASE64_PSBT")
-                        .help("Sets the PSBT to extract")
-                        .takes_value(true)
-                        .required(true)
-                        .number_of_values(1))
-                )
-        .subcommand(
-            SubCommand::with_name("finalize_psbt")
-                .about("Finalizes a psbt")
-                .arg(
-                    Arg::with_name("psbt")
-                        .long("psbt")
-                        .value_name("BASE64_PSBT")
-                        .help("Sets the PSBT to finalize")
-                        .takes_value(true)
-                        .required(true)
-                        .number_of_values(1))
-                .arg(
-                    Arg::with_name("assume_height")
-                        .long("assume_height")
-                        .value_name("HEIGHT")
-                        .help("Assume the blockchain has reached a specific height")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .required(false))
-                )
-        .subcommand(
-            SubCommand::with_name("combine_psbt")
-                .about("Combines multiple PSBTs into one")
-                .arg(
-                    Arg::with_name("psbt")
-                        .long("psbt")
-                        .value_name("BASE64_PSBT")
-                        .help("Add one PSBT to comine. This option can be repeated multiple times, one for each PSBT")
-                        .takes_value(true)
-                        .number_of_values(1)
-                        .required(true)
-                        .multiple(true))
-                )
-}
-
-pub fn add_global_flags<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
-    app.arg(
-        Arg::with_name("network")
-            .short("n")
-            .long("network")
-            .value_name("NETWORK")
-            .help("Sets the network")
-            .takes_value(true)
-            .default_value("testnet")
-            .possible_values(&["testnet", "regtest"]),
-    )
-    .arg(
-        Arg::with_name("wallet")
-            .short("w")
-            .long("wallet")
-            .value_name("WALLET_NAME")
-            .help("Selects the wallet to use")
-            .takes_value(true)
-            .default_value("main"),
-    )
-    .arg(
-        Arg::with_name("server")
-            .short("s")
-            .long("server")
-            .value_name("SERVER:PORT")
-            .help("Sets the Electrum server to use")
-            .takes_value(true)
-            .default_value("ssl://electrum.blockstream.info:60002"),
-    )
-    .arg(
-        Arg::with_name("proxy")
-            .short("p")
-            .long("proxy")
-            .value_name("SERVER:PORT")
-            .help("Sets the SOCKS5 proxy for the Electrum client")
-            .takes_value(true),
-    )
-    .arg(
-        Arg::with_name("descriptor")
-            .short("d")
-            .long("descriptor")
-            .value_name("DESCRIPTOR")
-            .help("Sets the descriptor to use for the external addresses")
-            .required(true)
-            .takes_value(true),
-    )
-    .arg(
-        Arg::with_name("change_descriptor")
-            .short("c")
-            .long("change_descriptor")
-            .value_name("DESCRIPTOR")
-            .help("Sets the descriptor to use for internal addresses")
-            .takes_value(true),
-    )
-    .arg(
-        Arg::with_name("v")
-            .short("v")
-            .multiple(true)
-            .help("Sets the level of verbosity"),
-    )
-    .subcommand(SubCommand::with_name("repl").about("Opens an interactive shell"))
-}
-
-#[maybe_async]
-pub fn handle_matches<C, D>(
-    wallet: &Wallet<C, D>,
-    matches: ArgMatches<'_>,
-) -> Result<serde_json::Value, Error>
-where
-    C: crate::blockchain::OnlineBlockchain,
-    D: crate::database::BatchDatabase,
-{
-    if let Some(_sub_matches) = matches.subcommand_matches("get_new_address") {
-        Ok(json!({
-            "address": wallet.get_new_address()?
-        }))
-    } else if let Some(_sub_matches) = matches.subcommand_matches("sync") {
-        maybe_await!(wallet.sync(log_progress(), None))?;
-        Ok(json!({}))
-    } else if let Some(_sub_matches) = matches.subcommand_matches("list_unspent") {
-        Ok(serde_json::to_value(&wallet.list_unspent()?)?)
-    } else if let Some(_sub_matches) = matches.subcommand_matches("list_transactions") {
-        Ok(serde_json::to_value(&wallet.list_transactions(false)?)?)
-    } else if let Some(_sub_matches) = matches.subcommand_matches("get_balance") {
-        Ok(json!({
-            "satoshi": wallet.get_balance()?
-        }))
-    } else if let Some(sub_matches) = matches.subcommand_matches("create_tx") {
-        let recipients = sub_matches
-            .values_of("to")
-            .unwrap()
-            .map(|s| parse_recipient(s))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|s| Error::Generic(s))?;
-        let mut tx_builder = TxBuilder::with_recipients(recipients);
-
-        if sub_matches.is_present("send_all") {
-            tx_builder = tx_builder.send_all();
-        }
-        if sub_matches.is_present("enable_rbf") {
-            tx_builder = tx_builder.enable_rbf();
-        }
-
-        if let Some(fee_rate) = sub_matches.value_of("fee_rate") {
-            let fee_rate = f32::from_str(fee_rate).map_err(|s| Error::Generic(s.to_string()))?;
-            tx_builder = tx_builder.fee_rate(FeeRate::from_sat_per_vb(fee_rate));
-        }
-        if let Some(utxos) = sub_matches.values_of("utxos") {
-            let utxos = utxos
-                .map(|i| parse_outpoint(i))
-                .collect::<Result<Vec<_>, _>>()
-                .map_err(|s| Error::Generic(s.to_string()))?;
-            tx_builder = tx_builder.utxos(utxos);
-        }
-
-        if let Some(unspendable) = sub_matches.values_of("unspendable") {
-            let unspendable = unspendable
-                .map(|i| parse_outpoint(i))
-                .collect::<Result<Vec<_>, _>>()
-                .map_err(|s| Error::Generic(s.to_string()))?;
-            tx_builder = tx_builder.unspendable(unspendable);
-        }
-        if let Some(policy) = sub_matches.value_of("policy") {
-            let policy = serde_json::from_str::<BTreeMap<String, Vec<usize>>>(&policy)
-                .map_err(|s| Error::Generic(s.to_string()))?;
-            tx_builder = tx_builder.policy_path(policy);
-        }
-
-        let (psbt, details) = wallet.create_tx(tx_builder)?;
-        Ok(json!({
-            "psbt": base64::encode(&serialize(&psbt)),
-            "details": details,
-        }))
-    } else if let Some(sub_matches) = matches.subcommand_matches("bump_fee") {
-        let txid = Txid::from_str(sub_matches.value_of("txid").unwrap())
-            .map_err(|s| Error::Generic(s.to_string()))?;
-
-        let fee_rate = f32::from_str(sub_matches.value_of("fee_rate").unwrap())
-            .map_err(|s| Error::Generic(s.to_string()))?;
-        let mut tx_builder = TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(fee_rate));
-
-        if sub_matches.is_present("send_all") {
-            tx_builder = tx_builder.send_all();
-        }
-
-        if let Some(utxos) = sub_matches.values_of("utxos") {
-            let utxos = utxos
-                .map(|i| parse_outpoint(i))
-                .collect::<Result<Vec<_>, _>>()
-                .map_err(|s| Error::Generic(s.to_string()))?;
-            tx_builder = tx_builder.utxos(utxos);
-        }
-
-        if let Some(unspendable) = sub_matches.values_of("unspendable") {
-            let unspendable = unspendable
-                .map(|i| parse_outpoint(i))
-                .collect::<Result<Vec<_>, _>>()
-                .map_err(|s| Error::Generic(s.to_string()))?;
-            tx_builder = tx_builder.unspendable(unspendable);
-        }
-
-        let (psbt, details) = wallet.bump_fee(&txid, tx_builder)?;
-        Ok(json!({
-            "psbt": base64::encode(&serialize(&psbt)),
-            "details": details,
-        }))
-    } else if let Some(_sub_matches) = matches.subcommand_matches("policies") {
-        Ok(json!({
-            "external": wallet.policies(ScriptType::External)?,
-            "internal": wallet.policies(ScriptType::Internal)?,
-        }))
-    } else if let Some(_sub_matches) = matches.subcommand_matches("public_descriptor") {
-        Ok(json!({
-            "external": wallet.public_descriptor(ScriptType::External)?.map(|d| d.to_string()),
-            "internal": wallet.public_descriptor(ScriptType::Internal)?.map(|d| d.to_string()),
-        }))
-    } else if let Some(sub_matches) = matches.subcommand_matches("sign") {
-        let psbt = base64::decode(sub_matches.value_of("psbt").unwrap()).unwrap();
-        let psbt: PartiallySignedTransaction = deserialize(&psbt).unwrap();
-        let assume_height = sub_matches
-            .value_of("assume_height")
-            .and_then(|s| Some(s.parse().unwrap()));
-        let (psbt, finalized) = wallet.sign(psbt, assume_height)?;
-        Ok(json!({
-            "psbt": base64::encode(&serialize(&psbt)),
-            "is_finalized": finalized,
-        }))
-    } else if let Some(sub_matches) = matches.subcommand_matches("broadcast") {
-        let tx = if sub_matches.value_of("psbt").is_some() {
-            let psbt = base64::decode(&sub_matches.value_of("psbt").unwrap()).unwrap();
-            let psbt: PartiallySignedTransaction = deserialize(&psbt).unwrap();
-            psbt.extract_tx()
-        } else if sub_matches.value_of("tx").is_some() {
-            deserialize(&Vec::<u8>::from_hex(&sub_matches.value_of("tx").unwrap()).unwrap())
-                .unwrap()
-        } else {
-            panic!("Missing `psbt` and `tx` option");
-        };
-
-        let txid = maybe_await!(wallet.broadcast(tx))?;
-        Ok(json!({ "txid": txid }))
-    } else if let Some(sub_matches) = matches.subcommand_matches("extract_psbt") {
-        let psbt = base64::decode(&sub_matches.value_of("psbt").unwrap()).unwrap();
-        let psbt: PartiallySignedTransaction = deserialize(&psbt).unwrap();
-        Ok(json!({
-            "raw_tx": serialize_hex(&psbt.extract_tx()),
-        }))
-    } else if let Some(sub_matches) = matches.subcommand_matches("finalize_psbt") {
-        let psbt = base64::decode(&sub_matches.value_of("psbt").unwrap()).unwrap();
-        let psbt: PartiallySignedTransaction = deserialize(&psbt).unwrap();
-
-        let assume_height = sub_matches
-            .value_of("assume_height")
-            .and_then(|s| Some(s.parse().unwrap()));
-
-        let (psbt, finalized) = wallet.finalize_psbt(psbt, assume_height)?;
-        Ok(json!({
-            "psbt": base64::encode(&serialize(&psbt)),
-            "is_finalized": finalized,
-        }))
-    } else if let Some(sub_matches) = matches.subcommand_matches("combine_psbt") {
-        let mut psbts = sub_matches
-            .values_of("psbt")
-            .unwrap()
-            .map(|s| {
-                let psbt = base64::decode(&s).unwrap();
-                let psbt: PartiallySignedTransaction = deserialize(&psbt).unwrap();
-
-                psbt
-            })
-            .collect::<Vec<_>>();
-
-        let init_psbt = psbts.pop().unwrap();
-        let final_psbt = psbts
-            .into_iter()
-            .try_fold::<_, _, Result<PartiallySignedTransaction, Error>>(
-                init_psbt,
-                |mut acc, x| {
-                    acc.merge(x)?;
-                    Ok(acc)
-                },
-            )?;
-
-        Ok(json!({ "psbt": base64::encode(&serialize(&final_psbt)) }))
-    } else {
-        Ok(serde_json::Value::Null)
-    }
-}

src/database/any.rs

@@ -0,0 +1,366 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Runtime-checked database types
+//!
+//! This module provides the implementation of [`AnyDatabase`] which allows switching the
+//! inner [`Database`] type at runtime.
+//!
+//! ## Example
+//!
+//! In this example, `wallet_memory` and `wallet_sled` have the same type of `Wallet<(), AnyDatabase>`.
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::database::{AnyDatabase, MemoryDatabase};
+//! # use bdk::{Wallet};
+//! let memory = MemoryDatabase::default();
+//! let wallet_memory = Wallet::new_offline("...", None, Network::Testnet, memory)?;
+//!
+//! # #[cfg(feature = "key-value-db")]
+//! # {
+//! let sled = sled::open("my-database")?.open_tree("default_tree")?;
+//! let wallet_sled = Wallet::new_offline("...", None, Network::Testnet, sled)?;
+//! # }
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! When paired with the use of [`ConfigurableDatabase`], it allows creating wallets with any
+//! database supported using a single line of code:
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::database::*;
+//! # use bdk::{Wallet};
+//! let config = serde_json::from_str("...")?;
+//! let database = AnyDatabase::from_config(&config)?;
+//! let wallet = Wallet::new_offline("...", None, Network::Testnet, database)?;
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use super::*;
+
+macro_rules! impl_from {
+    ( $from:ty, $to:ty, $variant:ident, $( $cfg:tt )* ) => {
+        $( $cfg )*
+        impl From<$from> for $to {
+            fn from(inner: $from) -> Self {
+                <$to>::$variant(inner)
+            }
+        }
+    };
+}
+
+macro_rules! impl_inner_method {
+    ( $enum_name:ident, $self:expr, $name:ident $(, $args:expr)* ) => {
+        match $self {
+            $enum_name::Memory(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "key-value-db")]
+            $enum_name::Sled(inner) => inner.$name( $($args, )* ),
+        }
+    }
+}
+
+/// Type that can contain any of the [`Database`] types defined by the library
+///
+/// It allows switching database type at runtime.
+///
+/// See [this module](crate::database::any)'s documentation for a usage example.
+#[derive(Debug)]
+pub enum AnyDatabase {
+    /// In-memory ephemeral database
+    Memory(memory::MemoryDatabase),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(sled::Tree),
+}
+
+impl_from!(memory::MemoryDatabase, AnyDatabase, Memory,);
+impl_from!(sled::Tree, AnyDatabase, Sled, #[cfg(feature = "key-value-db")]);
+
+/// Type that contains any of the [`BatchDatabase::Batch`] types defined by the library
+pub enum AnyBatch {
+    /// In-memory ephemeral database
+    Memory(<memory::MemoryDatabase as BatchDatabase>::Batch),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(<sled::Tree as BatchDatabase>::Batch),
+}
+
+impl_from!(
+    <memory::MemoryDatabase as BatchDatabase>::Batch,
+    AnyBatch,
+    Memory,
+);
+impl_from!(<sled::Tree as BatchDatabase>::Batch, AnyBatch, Sled, #[cfg(feature = "key-value-db")]);
+
+impl BatchOperations for AnyDatabase {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            set_script_pubkey,
+            script,
+            keychain,
+            child
+        )
+    }
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_utxo, utxo)
+    }
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_raw_tx, transaction)
+    }
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_tx, transaction)
+    }
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_last_index, keychain, value)
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            del_script_pubkey_from_path,
+            keychain,
+            child
+        )
+    }
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_path_from_script_pubkey, script)
+    }
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_utxo, outpoint)
+    }
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_raw_tx, txid)
+    }
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_tx, txid, include_raw)
+    }
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_last_index, keychain)
+    }
+}
+
+impl Database for AnyDatabase {
+    fn check_descriptor_checksum<B: AsRef<[u8]>>(
+        &mut self,
+        keychain: KeychainKind,
+        bytes: B,
+    ) -> Result<(), Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            check_descriptor_checksum,
+            keychain,
+            bytes
+        )
+    }
+
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_script_pubkeys, keychain)
+    }
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_utxos)
+    }
+    fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_raw_txs)
+    }
+    fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_txs, include_raw)
+    }
+
+    fn get_script_pubkey_from_path(
+        &self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            get_script_pubkey_from_path,
+            keychain,
+            child
+        )
+    }
+    fn get_path_from_script_pubkey(
+        &self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_path_from_script_pubkey, script)
+    }
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_utxo, outpoint)
+    }
+    fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_raw_tx, txid)
+    }
+    fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_tx, txid, include_raw)
+    }
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_last_index, keychain)
+    }
+
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        impl_inner_method!(AnyDatabase, self, increment_last_index, keychain)
+    }
+}
+
+impl BatchOperations for AnyBatch {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_script_pubkey, script, keychain, child)
+    }
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_utxo, utxo)
+    }
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_raw_tx, transaction)
+    }
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_tx, transaction)
+    }
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_last_index, keychain, value)
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(AnyBatch, self, del_script_pubkey_from_path, keychain, child)
+    }
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyBatch, self, del_path_from_script_pubkey, script)
+    }
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        impl_inner_method!(AnyBatch, self, del_utxo, outpoint)
+    }
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyBatch, self, del_raw_tx, txid)
+    }
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyBatch, self, del_tx, txid, include_raw)
+    }
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyBatch, self, del_last_index, keychain)
+    }
+}
+
+impl BatchDatabase for AnyDatabase {
+    type Batch = AnyBatch;
+
+    fn begin_batch(&self) -> Self::Batch {
+        match self {
+            AnyDatabase::Memory(inner) => inner.begin_batch().into(),
+            #[cfg(feature = "key-value-db")]
+            AnyDatabase::Sled(inner) => inner.begin_batch().into(),
+        }
+    }
+    fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> {
+        match self {
+            AnyDatabase::Memory(db) => match batch {
+                AnyBatch::Memory(batch) => db.commit_batch(batch),
+                #[cfg(feature = "key-value-db")]
+                _ => unimplemented!("Sled batch shouldn't be used with Memory db."),
+            },
+            #[cfg(feature = "key-value-db")]
+            AnyDatabase::Sled(db) => match batch {
+                AnyBatch::Sled(batch) => db.commit_batch(batch),
+                _ => unimplemented!("Memory batch shouldn't be used with Sled db."),
+            },
+        }
+    }
+}
+
+/// Configuration type for a [`sled::Tree`] database
+#[cfg(feature = "key-value-db")]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct SledDbConfiguration {
+    /// Main directory of the db
+    pub path: String,
+    /// Name of the database tree, a separated namespace for the data
+    pub tree_name: String,
+}
+
+#[cfg(feature = "key-value-db")]
+impl ConfigurableDatabase for sled::Tree {
+    type Config = SledDbConfiguration;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(sled::open(&config.path)?.open_tree(&config.tree_name)?)
+    }
+}
+
+/// Type that can contain any of the database configurations defined by the library
+///
+/// This allows storing a single configuration that can be loaded into an [`AnyDatabase`]
+/// instance. Wallets that plan to offer users the ability to switch blockchain backend at runtime
+/// will find this particularly useful.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub enum AnyDatabaseConfig {
+    /// Memory database has no config
+    Memory(()),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(SledDbConfiguration),
+}
+
+impl ConfigurableDatabase for AnyDatabase {
+    type Config = AnyDatabaseConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(match config {
+            AnyDatabaseConfig::Memory(inner) => {
+                AnyDatabase::Memory(memory::MemoryDatabase::from_config(inner)?)
+            }
+            #[cfg(feature = "key-value-db")]
+            AnyDatabaseConfig::Sled(inner) => AnyDatabase::Sled(sled::Tree::from_config(inner)?),
+        })
+    }
+}
+
+impl_from!((), AnyDatabaseConfig, Memory,);
+impl_from!(SledDbConfiguration, AnyDatabaseConfig, Sled, #[cfg(feature = "key-value-db")]);

src/database/keyvalue.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::convert::TryInto;
 
@@ -37,13 +24,13 @@ use crate::types::*;
 
 macro_rules! impl_batch_operations {
     ( { $($after_insert:tt)* }, $process_delete:ident ) => {
-        fn set_script_pubkey(&mut self, script: &Script, script_type: ScriptType, path: u32) -> Result<(), Error> {
-            let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        fn set_script_pubkey(&mut self, script: &Script, keychain: KeychainKind, path: u32) -> Result<(), Error> {
+            let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
             self.insert(key, serialize(script))$($after_insert)*;
 
             let key = MapKey::Script(Some(script)).as_map_key();
             let value = json!({
-                "t": script_type,
+                "t": keychain,
                 "p": path,
             });
             self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
@@ -51,11 +38,11 @@ macro_rules! impl_batch_operations {
             Ok(())
         }
 
-        fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
-            let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
+        fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+            let key = MapKey::Utxo(Some(&utxo.outpoint)).as_map_key();
             let value = json!({
                 "t": utxo.txout,
-                "i": utxo.is_internal,
+                "i": utxo.keychain,
             });
             self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
 
@@ -88,22 +75,22 @@ macro_rules! impl_batch_operations {
             Ok(())
         }
 
-        fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error> {
-            let key = MapKey::LastIndex(script_type).as_map_key();
+        fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+            let key = MapKey::LastIndex(keychain).as_map_key();
             self.insert(key, &value.to_be_bytes())$($after_insert)*;
 
             Ok(())
         }
 
-        fn del_script_pubkey_from_path(&mut self, script_type: ScriptType, path: u32) -> Result<Option<Script>, Error> {
-            let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        fn del_script_pubkey_from_path(&mut self, keychain: KeychainKind, path: u32) -> Result<Option<Script>, Error> {
+            let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
             Ok(res.map_or(Ok(None), |x| Some(deserialize(&x)).transpose())?)
         }
 
-        fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result<Option<(ScriptType, u32)>, Error> {
+        fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result<Option<(KeychainKind, u32)>, Error> {
             let key = MapKey::Script(Some(script)).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
@@ -120,8 +107,8 @@ macro_rules! impl_batch_operations {
             }
         }
 
-        fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-            let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+        fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+            let key = MapKey::Utxo(Some(outpoint)).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
@@ -130,9 +117,9 @@ macro_rules! impl_batch_operations {
                 Some(b) => {
                     let mut val: serde_json::Value = serde_json::from_slice(&b)?;
                     let txout = serde_json::from_value(val["t"].take())?;
-                    let is_internal = serde_json::from_value(val["i"].take())?;
+                    let keychain = serde_json::from_value(val["i"].take())?;
 
-                    Ok(Some(UTXO { outpoint: outpoint.clone(), txout, is_internal }))
+                    Ok(Some(LocalUtxo { outpoint: outpoint.clone(), txout, keychain }))
                 }
             }
         }
@@ -167,8 +154,8 @@ macro_rules! impl_batch_operations {
             }
         }
 
-        fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-            let key = MapKey::LastIndex(script_type).as_map_key();
+        fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+            let key = MapKey::LastIndex(keychain).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
@@ -206,10 +193,10 @@ impl BatchOperations for Batch {
 impl Database for Tree {
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error> {
-        let key = MapKey::DescriptorChecksum(script_type).as_map_key();
+        let key = MapKey::DescriptorChecksum(keychain).as_map_key();
 
         let prev = self.get(&key)?.map(|x| x.to_vec());
         if let Some(val) = prev {
@@ -224,8 +211,8 @@ impl Database for Tree {
         }
     }
 
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error> {
-        let key = MapKey::Path((script_type, None)).as_map_key();
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        let key = MapKey::Path((keychain, None)).as_map_key();
         self.scan_prefix(key)
             .map(|x| -> Result<_, Error> {
                 let (_, v) = x?;
@@ -234,8 +221,8 @@ impl Database for Tree {
             .collect()
     }
 
-    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error> {
-        let key = MapKey::UTXO(None).as_map_key();
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(None).as_map_key();
         self.scan_prefix(key)
             .map(|x| -> Result<_, Error> {
                 let (k, v) = x?;
@@ -243,12 +230,12 @@ impl Database for Tree {
 
                 let mut val: serde_json::Value = serde_json::from_slice(&v)?;
                 let txout = serde_json::from_value(val["t"].take())?;
-                let is_internal = serde_json::from_value(val["i"].take())?;
+                let keychain = serde_json::from_value(val["i"].take())?;
 
-                Ok(UTXO {
+                Ok(LocalUtxo {
                     outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 })
             })
             .collect()
@@ -282,17 +269,17 @@ impl Database for Tree {
 
     fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<Option<Script>, Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         Ok(self.get(key)?.map(|b| deserialize(&b)).transpose()?)
     }
 
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
@@ -305,18 +292,18 @@ impl Database for Tree {
             .transpose()
     }
 
-    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-        let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(Some(outpoint)).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
                 let mut val: serde_json::Value = serde_json::from_slice(&b)?;
                 let txout = serde_json::from_value(val["t"].take())?;
-                let is_internal = serde_json::from_value(val["i"].take())?;
+                let keychain = serde_json::from_value(val["i"].take())?;
 
-                Ok(UTXO {
-                    outpoint: outpoint.clone(),
+                Ok(LocalUtxo {
+                    outpoint: *outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 })
             })
             .transpose()
@@ -333,7 +320,7 @@ impl Database for Tree {
             .map(|b| -> Result<_, Error> {
                 let mut txdetails: TransactionDetails = serde_json::from_slice(&b)?;
                 if include_raw {
-                    txdetails.transaction = self.get_raw_tx(&txid)?;
+                    txdetails.transaction = self.get_raw_tx(txid)?;
                 }
 
                 Ok(txdetails)
@@ -341,8 +328,8 @@ impl Database for Tree {
             .transpose()
     }
 
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
                 let array: [u8; 4] = b
@@ -356,8 +343,8 @@ impl Database for Tree {
     }
 
     // inserts 0 if not present
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.update_and_fetch(key, |prev| {
             let new = match prev {
                 Some(b) => {

src/database/memory.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! In-memory ephemeral database
 //!
@@ -34,7 +21,7 @@ use bitcoin::consensus::encode::{deserialize, serialize};
 use bitcoin::hash_types::Txid;
 use bitcoin::{OutPoint, Script, Transaction};
 
-use crate::database::{BatchDatabase, BatchOperations, Database};
+use crate::database::{BatchDatabase, BatchOperations, ConfigurableDatabase, Database};
 use crate::error::Error;
 use crate::types::*;
 
@@ -47,17 +34,17 @@ use crate::types::*;
 // descriptor checksum  d{i,e} -> vec<u8>
 
 pub(crate) enum MapKey<'a> {
-    Path((Option<ScriptType>, Option<u32>)),
+    Path((Option<KeychainKind>, Option<u32>)),
     Script(Option<&'a Script>),
-    UTXO(Option<&'a OutPoint>),
+    Utxo(Option<&'a OutPoint>),
     RawTx(Option<&'a Txid>),
     Transaction(Option<&'a Txid>),
-    LastIndex(ScriptType),
-    DescriptorChecksum(ScriptType),
+    LastIndex(KeychainKind),
+    DescriptorChecksum(KeychainKind),
 }
 
 impl MapKey<'_> {
-    pub fn as_prefix(&self) -> Vec<u8> {
+    fn as_prefix(&self) -> Vec<u8> {
         match self {
             MapKey::Path((st, _)) => {
                 let mut v = b"p".to_vec();
@@ -67,7 +54,7 @@ impl MapKey<'_> {
                 v
             }
             MapKey::Script(_) => b"s".to_vec(),
-            MapKey::UTXO(_) => b"u".to_vec(),
+            MapKey::Utxo(_) => b"u".to_vec(),
             MapKey::RawTx(_) => b"r".to_vec(),
             MapKey::Transaction(_) => b"t".to_vec(),
             MapKey::LastIndex(st) => [b"c", st.as_ref()].concat(),
@@ -77,9 +64,9 @@ impl MapKey<'_> {
 
     fn serialize_content(&self) -> Vec<u8> {
         match self {
-            MapKey::Path((_, Some(child))) => u32::from(*child).to_be_bytes().to_vec(),
+            MapKey::Path((_, Some(child))) => child.to_be_bytes().to_vec(),
             MapKey::Script(Some(s)) => serialize(*s),
-            MapKey::UTXO(Some(s)) => serialize(*s),
+            MapKey::Utxo(Some(s)) => serialize(*s),
             MapKey::RawTx(Some(s)) => serialize(*s),
             MapKey::Transaction(Some(s)) => serialize(*s),
             _ => vec![],
@@ -94,8 +81,8 @@ impl MapKey<'_> {
     }
 }
 
-fn after(key: &Vec<u8>) -> Vec<u8> {
-    let mut key = key.clone();
+fn after(key: &[u8]) -> Vec<u8> {
+    let mut key = key.to_owned();
     let mut idx = key.len();
     while idx > 0 {
         if key[idx - 1] == 0xFF {
@@ -141,15 +128,15 @@ impl BatchOperations for MemoryDatabase {
     fn set_script_pubkey(
         &mut self,
         script: &Script,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<(), Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         self.map.insert(key, Box::new(script.clone()));
 
         let key = MapKey::Script(Some(script)).as_map_key();
         let value = json!({
-            "t": script_type,
+            "t": keychain,
             "p": path,
         });
         self.map.insert(key, Box::new(value));
@@ -157,10 +144,10 @@ impl BatchOperations for MemoryDatabase {
         Ok(())
     }
 
-    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
-        let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
+        let key = MapKey::Utxo(Some(&utxo.outpoint)).as_map_key();
         self.map
-            .insert(key, Box::new((utxo.txout.clone(), utxo.is_internal)));
+            .insert(key, Box::new((utxo.txout.clone(), utxo.keychain)));
 
         Ok(())
     }
@@ -186,8 +173,8 @@ impl BatchOperations for MemoryDatabase {
 
         Ok(())
     }
-    fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.map.insert(key, Box::new(value));
 
         Ok(())
@@ -195,10 +182,10 @@ impl BatchOperations for MemoryDatabase {
 
     fn del_script_pubkey_from_path(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<Option<Script>, Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
@@ -207,7 +194,7 @@ impl BatchOperations for MemoryDatabase {
     fn del_path_from_script_pubkey(
         &mut self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
@@ -223,19 +210,19 @@ impl BatchOperations for MemoryDatabase {
             }
         }
     }
-    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-        let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(Some(outpoint)).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
         match res {
             None => Ok(None),
             Some(b) => {
-                let (txout, is_internal) = b.downcast_ref().cloned().unwrap();
-                Ok(Some(UTXO {
-                    outpoint: outpoint.clone(),
+                let (txout, keychain) = b.downcast_ref().cloned().unwrap();
+                Ok(Some(LocalUtxo {
+                    outpoint: *outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 }))
             }
         }
@@ -272,8 +259,8 @@ impl BatchOperations for MemoryDatabase {
             }
         }
     }
-    fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
@@ -287,10 +274,10 @@ impl BatchOperations for MemoryDatabase {
 impl Database for MemoryDatabase {
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error> {
-        let key = MapKey::DescriptorChecksum(script_type).as_map_key();
+        let key = MapKey::DescriptorChecksum(keychain).as_map_key();
 
         let prev = self
             .map
@@ -308,25 +295,25 @@ impl Database for MemoryDatabase {
         }
     }
 
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error> {
-        let key = MapKey::Path((script_type, None)).as_map_key();
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        let key = MapKey::Path((keychain, None)).as_map_key();
         self.map
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(_, v)| Ok(v.downcast_ref().cloned().unwrap()))
             .collect()
     }
 
-    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error> {
-        let key = MapKey::UTXO(None).as_map_key();
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(None).as_map_key();
         self.map
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(k, v)| {
                 let outpoint = deserialize(&k[1..]).unwrap();
-                let (txout, is_internal) = v.downcast_ref().cloned().unwrap();
-                Ok(UTXO {
+                let (txout, keychain) = v.downcast_ref().cloned().unwrap();
+                Ok(LocalUtxo {
                     outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 })
             })
             .collect()
@@ -358,10 +345,10 @@ impl Database for MemoryDatabase {
 
     fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<Option<Script>, Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         Ok(self
             .map
             .get(&key)
@@ -371,7 +358,7 @@ impl Database for MemoryDatabase {
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
             let mut val: serde_json::Value = b.downcast_ref().cloned().unwrap();
@@ -382,14 +369,14 @@ impl Database for MemoryDatabase {
         }))
     }
 
-    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
-        let key = MapKey::UTXO(Some(outpoint)).as_map_key();
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
+        let key = MapKey::Utxo(Some(outpoint)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
-            let (txout, is_internal) = b.downcast_ref().cloned().unwrap();
-            UTXO {
-                outpoint: outpoint.clone(),
+            let (txout, keychain) = b.downcast_ref().cloned().unwrap();
+            LocalUtxo {
+                outpoint: *outpoint,
                 txout,
-                is_internal,
+                keychain,
             }
         }))
     }
@@ -407,26 +394,26 @@ impl Database for MemoryDatabase {
         Ok(self.map.get(&key).map(|b| {
             let mut txdetails: TransactionDetails = b.downcast_ref().cloned().unwrap();
             if include_raw {
-                txdetails.transaction = self.get_raw_tx(&txid).unwrap();
+                txdetails.transaction = self.get_raw_tx(txid).unwrap();
             }
 
             txdetails
         }))
     }
 
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         Ok(self.map.get(&key).map(|b| *b.downcast_ref().unwrap()))
     }
 
     // inserts 0 if not present
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         let value = self
             .map
-            .entry(key.clone())
+            .entry(key)
             .and_modify(|x| *x.downcast_mut::<u32>().unwrap() += 1)
-            .or_insert(Box::<u32>::new(0))
+            .or_insert_with(|| Box::<u32>::new(0))
             .downcast_mut()
             .unwrap();
 
@@ -442,24 +429,33 @@ impl BatchDatabase for MemoryDatabase {
     }
 
     fn commit_batch(&mut self, mut batch: Self::Batch) -> Result<(), Error> {
-        for key in batch.deleted_keys {
-            self.map.remove(&key);
+        for key in batch.deleted_keys.iter() {
+            self.map.remove(key);
         }
-
-        Ok(self.map.append(&mut batch.map))
+        self.map.append(&mut batch.map);
+        Ok(())
     }
 }
 
-#[cfg(test)]
-impl MemoryDatabase {
-    // Artificially insert a tx in the database, as if we had found it with a `sync`
-    pub fn received_tx(
-        &mut self,
-        tx_meta: testutils::TestIncomingTx,
-        current_height: Option<u32>,
-    ) -> bitcoin::Txid {
-        use std::str::FromStr;
+impl ConfigurableDatabase for MemoryDatabase {
+    type Config = ();
 
+    fn from_config(_config: &Self::Config) -> Result<Self, Error> {
+        Ok(MemoryDatabase::default())
+    }
+}
+
+#[macro_export]
+#[doc(hidden)]
+/// Artificially insert a tx in the database, as if we had found it with a `sync`. This is a hidden
+/// macro and not a `[cfg(test)]` function so it can be called within the context of doctests which
+/// don't have `test` set.
+macro_rules! populate_test_db {
+    ($db:expr, $tx_meta:expr, $current_height:expr$(,)?) => {{
+        use $crate::database::BatchOperations;
+        let mut db = $db;
+        let tx_meta = $tx_meta;
+        let current_height: Option<u32> = $current_height;
         let tx = Transaction {
             version: 1,
             lock_time: 0,
@@ -477,35 +473,66 @@ impl MemoryDatabase {
         };
 
         let txid = tx.txid();
-        let height = tx_meta
-            .min_confirmations
-            .map(|conf| current_height.unwrap().checked_sub(conf as u32).unwrap());
+        let confirmation_time = tx_meta.min_confirmations.map(|conf| ConfirmationTime {
+            height: current_height.unwrap().checked_sub(conf as u32).unwrap(),
+            timestamp: 0,
+        });
 
         let tx_details = TransactionDetails {
             transaction: Some(tx.clone()),
             txid,
-            timestamp: 0,
-            height,
+            fee: Some(0),
             received: 0,
             sent: 0,
-            fees: 0,
+            confirmation_time,
+            verified: current_height.is_some(),
         };
 
-        self.set_tx(&tx_details).unwrap();
+        db.set_tx(&tx_details).unwrap();
         for (vout, out) in tx.output.iter().enumerate() {
-            self.set_utxo(&UTXO {
+            db.set_utxo(&LocalUtxo {
                 txout: out.clone(),
                 outpoint: OutPoint {
                     txid,
                     vout: vout as u32,
                 },
-                is_internal: false,
+                keychain: KeychainKind::External,
             })
             .unwrap();
         }
 
         txid
-    }
+    }};
+}
+
+#[macro_export]
+#[doc(hidden)]
+/// Macro for getting a wallet for use in a doctest
+macro_rules! doctest_wallet {
+    () => {{
+        use $crate::bitcoin::Network;
+        use $crate::database::MemoryDatabase;
+        use $crate::testutils;
+        let descriptor = "wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)";
+        let descriptors = testutils!(@descriptors (descriptor) (descriptor));
+
+        let mut db = MemoryDatabase::new();
+        let txid = populate_test_db!(
+            &mut db,
+            testutils! {
+                @tx ( (@external descriptors, 0) => 500_000 ) (@confirmations 1)
+            },
+            Some(100),
+        );
+
+        $crate::Wallet::new_offline(
+            &descriptors.0,
+            descriptors.1.as_ref(),
+            Network::Regtest,
+            db
+        )
+        .unwrap()
+    }}
 }
 
 #[cfg(test)]

src/database/mod.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Database types
 //!
@@ -43,6 +30,9 @@ use bitcoin::{OutPoint, Script, Transaction, TxOut};
 use crate::error::Error;
 use crate::types::*;
 
+pub mod any;
+pub use any::{AnyDatabase, AnyDatabaseConfig};
+
 #[cfg(feature = "key-value-db")]
 pub(crate) mod keyvalue;
 
@@ -54,36 +44,36 @@ pub use memory::MemoryDatabase;
 /// This trait defines the list of operations that must be implemented on the [`Database`] type and
 /// the [`BatchDatabase::Batch`] type.
 pub trait BatchOperations {
-    /// Store a script_pubkey along with its script type and child number
+    /// Store a script_pubkey along with its keychain and child number.
     fn set_script_pubkey(
         &mut self,
         script: &Script,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         child: u32,
     ) -> Result<(), Error>;
-    /// Store a [`UTXO`]
-    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error>;
+    /// Store a [`LocalUtxo`]
+    fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error>;
     /// Store a raw transaction
     fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error>;
     /// Store the metadata of a transaction
     fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error>;
-    /// Store the last derivation index for a given script type
-    fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error>;
+    /// Store the last derivation index for a given keychain.
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error>;
 
-    /// Delete a script_pubkey given the script type and its child number
+    /// Delete a script_pubkey given the keychain and its child number.
     fn del_script_pubkey_from_path(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         child: u32,
     ) -> Result<Option<Script>, Error>;
-    /// Delete the data related to a specific script_pubkey, meaning the script type and the child
-    /// number
+    /// Delete the data related to a specific script_pubkey, meaning the keychain and the child
+    /// number.
     fn del_path_from_script_pubkey(
         &mut self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error>;
-    /// Delete a [`UTXO`] given its [`OutPoint`]
-    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
+    ) -> Result<Option<(KeychainKind, u32)>, Error>;
+    /// Delete a [`LocalUtxo`] given its [`OutPoint`]
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error>;
     /// Delete a raw transaction given its [`Txid`]
     fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error>;
     /// Delete the metadata of a transaction and optionally the raw transaction itself
@@ -92,58 +82,58 @@ pub trait BatchOperations {
         txid: &Txid,
         include_raw: bool,
     ) -> Result<Option<TransactionDetails>, Error>;
-    /// Delete the last derivation index for a script type
-    fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error>;
+    /// Delete the last derivation index for a keychain.
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error>;
 }
 
 /// Trait for reading data from a database
 ///
 /// This traits defines the operations that can be used to read data out of a database
 pub trait Database: BatchOperations {
-    /// Read and checks the descriptor checksum for a given script type
+    /// Read and checks the descriptor checksum for a given keychain.
     ///
     /// Should return [`Error::ChecksumMismatch`](crate::error::Error::ChecksumMismatch) if the
     /// checksum doesn't match. If there's no checksum in the database, simply store it for the
     /// next time.
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error>;
 
     /// Return the list of script_pubkeys
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error>;
-    /// Return the list of [`UTXO`]s
-    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error>;
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error>;
+    /// Return the list of [`LocalUtxo`]s
+    fn iter_utxos(&self) -> Result<Vec<LocalUtxo>, Error>;
     /// Return the list of raw transactions
     fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error>;
     /// Return the list of transactions metadata
     fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error>;
 
-    /// Fetch a script_pubkey given the script type and child number
+    /// Fetch a script_pubkey given the child number of a keychain.
     fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         child: u32,
     ) -> Result<Option<Script>, Error>;
-    /// Fetch the script type and child number of a given script_pubkey
+    /// Fetch the keychain and child number of a given script_pubkey
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error>;
-    /// Fetch a [`UTXO`] given its [`OutPoint`]
-    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
+    ) -> Result<Option<(KeychainKind, u32)>, Error>;
+    /// Fetch a [`LocalUtxo`] given its [`OutPoint`]
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error>;
     /// Fetch a raw transaction given its [`Txid`]
     fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error>;
     /// Fetch the transaction metadata and optionally also the raw transaction
     fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error>;
-    /// Return the last defivation index for a script type
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error>;
+    /// Return the last defivation index for a keychain.
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error>;
 
-    /// Increment the last derivation index for a script type and returns it
+    /// Increment the last derivation index for a keychain and return it
     ///
     /// It should insert and return `0` if not present in the database
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error>;
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error>;
 }
 
 /// Trait for a database that supports batch operations
@@ -159,27 +149,36 @@ pub trait BatchDatabase: Database {
     fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error>;
 }
 
+/// Trait for [`Database`] types that can be created given a configuration
+pub trait ConfigurableDatabase: Database + Sized {
+    /// Type that contains the configuration
+    type Config: std::fmt::Debug;
+
+    /// Create a new instance given a configuration
+    fn from_config(config: &Self::Config) -> Result<Self, Error>;
+}
+
 pub(crate) trait DatabaseUtils: Database {
     fn is_mine(&self, script: &Script) -> Result<bool, Error> {
         self.get_path_from_script_pubkey(script)
             .map(|o| o.is_some())
     }
 
-    fn get_raw_tx_or<F>(&self, txid: &Txid, f: F) -> Result<Option<Transaction>, Error>
+    fn get_raw_tx_or<D>(&self, txid: &Txid, default: D) -> Result<Option<Transaction>, Error>
     where
-        F: FnOnce() -> Result<Option<Transaction>, Error>,
+        D: FnOnce() -> Result<Option<Transaction>, Error>,
     {
         self.get_tx(txid, true)?
             .map(|t| t.transaction)
             .flatten()
-            .map_or_else(f, |t| Ok(Some(t)))
+            .map_or_else(default, |t| Ok(Some(t)))
     }
 
     fn get_previous_output(&self, outpoint: &OutPoint) -> Result<Option<TxOut>, Error> {
         self.get_raw_tx(&outpoint.txid)?
             .map(|previous_tx| {
                 if outpoint.vout as usize >= previous_tx.output.len() {
-                    Err(Error::InvalidOutpoint(outpoint.clone()))
+                    Err(Error::InvalidOutpoint(*outpoint))
                 } else {
                     Ok(previous_tx.output[outpoint.vout as usize].clone())
                 }
@@ -205,17 +204,17 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        tree.set_script_pubkey(&script, script_type, path).unwrap();
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
 
         assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, path).unwrap(),
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
             Some(script.clone())
         );
         assert_eq!(
             tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
+            Some((keychain, path))
         );
     }
 
@@ -226,12 +225,12 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        batch.set_script_pubkey(&script, script_type, path).unwrap();
+        batch.set_script_pubkey(&script, keychain, path).unwrap();
 
         assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, path).unwrap(),
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
             None
         );
         assert_eq!(tree.get_path_from_script_pubkey(&script).unwrap(), None);
@@ -239,12 +238,12 @@ pub mod test {
         tree.commit_batch(batch).unwrap();
 
         assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, path).unwrap(),
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
             Some(script.clone())
         );
         assert_eq!(
             tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
+            Some((keychain, path))
         );
     }
 
@@ -253,9 +252,9 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        tree.set_script_pubkey(&script, script_type, path).unwrap();
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
 
         assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
     }
@@ -265,12 +264,12 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        tree.set_script_pubkey(&script, script_type, path).unwrap();
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
         assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
 
-        tree.del_script_pubkey_from_path(script_type, path).unwrap();
+        tree.del_script_pubkey_from_path(keychain, path).unwrap();
         assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0);
     }
 
@@ -286,10 +285,10 @@ pub mod test {
             value: 133742,
             script_pubkey: script,
         };
-        let utxo = UTXO {
+        let utxo = LocalUtxo {
             txout,
             outpoint,
-            is_internal: false,
+            keychain: KeychainKind::External,
         };
 
         tree.set_utxo(&utxo).unwrap();
@@ -315,11 +314,14 @@ pub mod test {
         let mut tx_details = TransactionDetails {
             transaction: Some(tx),
             txid,
-            timestamp: 123456,
             received: 1337,
             sent: 420420,
-            fees: 140,
-            height: Some(1000),
+            fee: Some(140),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 123456,
+                height: 1000,
+            }),
+            verified: true,
         };
 
         tree.set_tx(&tx_details).unwrap();
@@ -344,24 +346,27 @@ pub mod test {
     }
 
     pub fn test_last_index<D: Database>(mut tree: D) {
-        tree.set_last_index(ScriptType::External, 1337).unwrap();
+        tree.set_last_index(KeychainKind::External, 1337).unwrap();
 
         assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
+            tree.get_last_index(KeychainKind::External).unwrap(),
             Some(1337)
         );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), None);
+        assert_eq!(tree.get_last_index(KeychainKind::Internal).unwrap(), None);
 
-        let res = tree.increment_last_index(ScriptType::External).unwrap();
+        let res = tree.increment_last_index(KeychainKind::External).unwrap();
         assert_eq!(res, 1338);
-        let res = tree.increment_last_index(ScriptType::Internal).unwrap();
+        let res = tree.increment_last_index(KeychainKind::Internal).unwrap();
         assert_eq!(res, 0);
 
         assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
+            tree.get_last_index(KeychainKind::External).unwrap(),
             Some(1338)
         );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), Some(0));
+        assert_eq!(
+            tree.get_last_index(KeychainKind::Internal).unwrap(),
+            Some(0)
+        );
     }
 
     // TODO: more tests...

src/descriptor/checksum.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptor checksum
 //!
@@ -29,7 +16,7 @@
 
 use std::iter::FromIterator;
 
-use crate::descriptor::Error;
+use crate::descriptor::DescriptorError;
 
 const INPUT_CHARSET: &str =  "0123456789()[],'/*abcdefgh@:$%{}IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~ijklmnopqrstuvwxyzABCDEFGH`#\"\\ ";
 const CHECKSUM_CHARSET: &str = "qpzry9x8gf2tvdw0s3jn54khce6mua7l";
@@ -57,14 +44,14 @@ fn poly_mod(mut c: u64, val: u64) -> u64 {
 }
 
 /// Compute the checksum of a descriptor
-pub fn get_checksum(desc: &str) -> Result<String, Error> {
+pub fn get_checksum(desc: &str) -> Result<String, DescriptorError> {
     let mut c = 1;
     let mut cls = 0;
     let mut clscount = 0;
     for ch in desc.chars() {
         let pos = INPUT_CHARSET
             .find(ch)
-            .ok_or(Error::InvalidDescriptorCharacter(ch))? as u64;
+            .ok_or(DescriptorError::InvalidDescriptorCharacter(ch))? as u64;
         c = poly_mod(c, pos & 31);
         cls = cls * 3 + (pos >> 5);
         clscount += 1;
@@ -92,3 +79,35 @@ pub fn get_checksum(desc: &str) -> Result<String, Error> {
 
     Ok(String::from_iter(chars))
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::descriptor::get_checksum;
+
+    // test get_checksum() function; it should return the same value as Bitcoin Core
+    #[test]
+    fn test_get_checksum() {
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)";
+        assert_eq!(get_checksum(desc).unwrap(), "tqz0nc62");
+
+        let desc = "pkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/44'/1'/0'/0/*)";
+        assert_eq!(get_checksum(desc).unwrap(), "lasegmfs");
+    }
+
+    #[test]
+    fn test_get_checksum_invalid_character() {
+        let sparkle_heart = vec![240, 159, 146, 150];
+        let sparkle_heart = std::str::from_utf8(&sparkle_heart)
+            .unwrap()
+            .chars()
+            .next()
+            .unwrap();
+        let invalid_desc = format!("wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcL{}fjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)", sparkle_heart);
+
+        assert!(matches!(
+            get_checksum(&invalid_desc).err(),
+            Some(DescriptorError::InvalidDescriptorCharacter(invalid_char)) if invalid_char == sparkle_heart
+        ));
+    }
+}

src/descriptor/derived.rs

@@ -0,0 +1,150 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Derived descriptor keys
+
+use std::cmp::Ordering;
+use std::fmt;
+use std::hash::{Hash, Hasher};
+use std::ops::Deref;
+
+use bitcoin::hashes::hash160;
+use bitcoin::PublicKey;
+
+pub use miniscript::{
+    descriptor::KeyMap, descriptor::Wildcard, Descriptor, DescriptorPublicKey, Legacy, Miniscript,
+    ScriptContext, Segwitv0,
+};
+use miniscript::{MiniscriptKey, ToPublicKey, TranslatePk};
+
+use crate::wallet::utils::SecpCtx;
+
+/// Extended [`DescriptorPublicKey`] that has been derived
+///
+/// Derived keys are guaranteed to never contain wildcards of any kind
+#[derive(Debug, Clone)]
+pub struct DerivedDescriptorKey<'s>(DescriptorPublicKey, &'s SecpCtx);
+
+impl<'s> DerivedDescriptorKey<'s> {
+    /// Construct a new derived key
+    ///
+    /// Panics if the key is wildcard
+    pub fn new(key: DescriptorPublicKey, secp: &'s SecpCtx) -> DerivedDescriptorKey<'s> {
+        if let DescriptorPublicKey::XPub(xpub) = &key {
+            assert!(xpub.wildcard == Wildcard::None)
+        }
+
+        DerivedDescriptorKey(key, secp)
+    }
+}
+
+impl<'s> Deref for DerivedDescriptorKey<'s> {
+    type Target = DescriptorPublicKey;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl<'s> PartialEq for DerivedDescriptorKey<'s> {
+    fn eq(&self, other: &Self) -> bool {
+        self.0 == other.0
+    }
+}
+
+impl<'s> Eq for DerivedDescriptorKey<'s> {}
+
+impl<'s> PartialOrd for DerivedDescriptorKey<'s> {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        self.0.partial_cmp(&other.0)
+    }
+}
+
+impl<'s> Ord for DerivedDescriptorKey<'s> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.0.cmp(&other.0)
+    }
+}
+
+impl<'s> fmt::Display for DerivedDescriptorKey<'s> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl<'s> Hash for DerivedDescriptorKey<'s> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.0.hash(state);
+    }
+}
+
+impl<'s> MiniscriptKey for DerivedDescriptorKey<'s> {
+    type Hash = Self;
+
+    fn to_pubkeyhash(&self) -> Self::Hash {
+        DerivedDescriptorKey(self.0.to_pubkeyhash(), self.1)
+    }
+
+    fn is_uncompressed(&self) -> bool {
+        self.0.is_uncompressed()
+    }
+    fn serialized_len(&self) -> usize {
+        self.0.serialized_len()
+    }
+}
+
+impl<'s> ToPublicKey for DerivedDescriptorKey<'s> {
+    fn to_public_key(&self) -> PublicKey {
+        match &self.0 {
+            DescriptorPublicKey::SinglePub(ref spub) => spub.key.to_public_key(),
+            DescriptorPublicKey::XPub(ref xpub) => {
+                xpub.xkey
+                    .derive_pub(self.1, &xpub.derivation_path)
+                    .expect("Shouldn't fail, only normal derivations")
+                    .public_key
+            }
+        }
+    }
+
+    fn hash_to_hash160(hash: &Self::Hash) -> hash160::Hash {
+        hash.to_public_key().to_pubkeyhash()
+    }
+}
+
+pub(crate) trait AsDerived {
+    // Derive a descriptor and transform all of its keys to `DerivedDescriptorKey`
+    fn as_derived<'s>(&self, index: u32, secp: &'s SecpCtx)
+        -> Descriptor<DerivedDescriptorKey<'s>>;
+
+    // Transform the keys into `DerivedDescriptorKey`.
+    //
+    // Panics if the descriptor is not "fixed", i.e. if it's derivable
+    fn as_derived_fixed<'s>(&self, secp: &'s SecpCtx) -> Descriptor<DerivedDescriptorKey<'s>>;
+}
+
+impl AsDerived for Descriptor<DescriptorPublicKey> {
+    fn as_derived<'s>(
+        &self,
+        index: u32,
+        secp: &'s SecpCtx,
+    ) -> Descriptor<DerivedDescriptorKey<'s>> {
+        self.derive(index).translate_pk_infallible(
+            |key| DerivedDescriptorKey::new(key.clone(), secp),
+            |key| DerivedDescriptorKey::new(key.clone(), secp),
+        )
+    }
+
+    fn as_derived_fixed<'s>(&self, secp: &'s SecpCtx) -> Descriptor<DerivedDescriptorKey<'s>> {
+        assert!(!self.is_deriveable());
+
+        self.as_derived(0, secp)
+    }
+}

src/descriptor/error.rs

@@ -1,55 +1,58 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptor errors
 
 /// Errors related to the parsing and usage of descriptors
 #[derive(Debug)]
 pub enum Error {
-    InternalError,
-    InvalidPrefix(Vec<u8>),
-    HardenedDerivationOnXpub,
-    MalformedInput,
-    KeyParsingError(String),
+    /// Invalid HD Key path, such as having a wildcard but a length != 1
+    InvalidHdKeyPath,
+    /// The provided descriptor doesn't match its checksum
+    InvalidDescriptorChecksum,
+    /// The descriptor contains hardened derivation steps on public extended keys
+    HardenedDerivationXpub,
+    /// The descriptor contains multiple keys with the same BIP32 fingerprint
+    DuplicatedKeys,
 
+    /// Error thrown while working with [`keys`](crate::keys)
+    Key(crate::keys::KeyError),
+    /// Error while extracting and manipulating policies
     Policy(crate::descriptor::policy::PolicyError),
 
-    InputIndexDoesntExist,
-    MissingPublicKey,
-    MissingDetails,
-
+    /// Invalid character found in the descriptor checksum
     InvalidDescriptorCharacter(char),
 
-    CantDeriveWithMiniscript,
-
-    BIP32(bitcoin::util::bip32::Error),
+    /// BIP32 error
+    Bip32(bitcoin::util::bip32::Error),
+    /// Error during base58 decoding
     Base58(bitcoin::util::base58::Error),
-    PK(bitcoin::util::key::Error),
+    /// Key-related error
+    Pk(bitcoin::util::key::Error),
+    /// Miniscript error
     Miniscript(miniscript::Error),
+    /// Hex decoding error
     Hex(bitcoin::hashes::hex::Error),
 }
 
+impl From<crate::keys::KeyError> for Error {
+    fn from(key_error: crate::keys::KeyError) -> Error {
+        match key_error {
+            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
+            crate::keys::KeyError::Bip32(inner) => Error::Bip32(inner),
+            e => Error::Key(e),
+        }
+    }
+}
+
 impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{:?}", self)
@@ -58,9 +61,9 @@ impl std::fmt::Display for Error {
 
 impl std::error::Error for Error {}
 
-impl_error!(bitcoin::util::bip32::Error, BIP32);
+impl_error!(bitcoin::util::bip32::Error, Bip32);
 impl_error!(bitcoin::util::base58::Error, Base58);
-impl_error!(bitcoin::util::key::Error, PK);
+impl_error!(bitcoin::util::key::Error, Pk);
 impl_error!(miniscript::Error, Miniscript);
 impl_error!(bitcoin::hashes::hex::Error, Hex);
 impl_error!(crate::descriptor::policy::PolicyError, Policy);

src/descriptor/mod.rs

@@ -1,94 +1,292 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Descriptors
 //!
 //! This module contains generic utilities to work with descriptors, plus some re-exported types
 //! from [`miniscript`].
 
-use std::collections::{BTreeMap, HashMap};
-use std::fmt;
-use std::sync::Arc;
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::ops::Deref;
 
-use bitcoin::hashes::hash160;
-use bitcoin::secp256k1::Secp256k1;
-use bitcoin::util::bip32::{ChildNumber, DerivationPath, Fingerprint};
-use bitcoin::util::psbt;
-use bitcoin::{PublicKey, Script, TxOut};
-
-use miniscript::descriptor::{DescriptorPublicKey, DescriptorXKey, InnerXKey};
-pub use miniscript::{
-    Descriptor, Legacy, Miniscript, MiniscriptKey, ScriptContext, Segwitv0, Terminal, ToPublicKey,
+use bitcoin::util::bip32::{
+    ChildNumber, DerivationPath, ExtendedPrivKey, ExtendedPubKey, Fingerprint, KeySource,
 };
+use bitcoin::util::psbt;
+use bitcoin::{Network, PublicKey, Script, TxOut};
+
+use miniscript::descriptor::{DescriptorPublicKey, DescriptorType, DescriptorXKey, Wildcard};
+pub use miniscript::{descriptor::KeyMap, Descriptor, Legacy, Miniscript, ScriptContext, Segwitv0};
+use miniscript::{DescriptorTrait, ForEachKey, TranslatePk};
+
+use crate::descriptor::policy::BuildSatisfaction;
 
 pub mod checksum;
+pub(crate) mod derived;
+#[doc(hidden)]
+pub mod dsl;
 pub mod error;
 pub mod policy;
+pub mod template;
 
 pub use self::checksum::get_checksum;
-use self::error::Error;
+use self::derived::AsDerived;
+pub use self::derived::DerivedDescriptorKey;
+pub use self::error::Error as DescriptorError;
 pub use self::policy::Policy;
+use self::template::DescriptorTemplateOut;
+use crate::keys::{IntoDescriptorKey, KeyError};
 use crate::wallet::signer::SignersContainer;
+use crate::wallet::utils::SecpCtx;
 
 /// Alias for a [`Descriptor`] that can contain extended keys using [`DescriptorPublicKey`]
 pub type ExtendedDescriptor = Descriptor<DescriptorPublicKey>;
 
+/// Alias for a [`Descriptor`] that contains extended **derived** keys
+pub type DerivedDescriptor<'s> = Descriptor<DerivedDescriptorKey<'s>>;
+
 /// Alias for the type of maps that represent derivation paths in a [`psbt::Input`] or
 /// [`psbt::Output`]
 ///
 /// [`psbt::Input`]: bitcoin::util::psbt::Input
 /// [`psbt::Output`]: bitcoin::util::psbt::Output
-pub type HDKeyPaths = BTreeMap<PublicKey, (Fingerprint, DerivationPath)>;
+pub type HdKeyPaths = BTreeMap<PublicKey, KeySource>;
+
+/// Trait for types which can be converted into an [`ExtendedDescriptor`] and a [`KeyMap`] usable by a wallet in a specific [`Network`]
+pub trait IntoWalletDescriptor {
+    /// Convert to wallet descriptor
+    fn into_wallet_descriptor(
+        self,
+        secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError>;
+}
+
+impl IntoWalletDescriptor for &str {
+    fn into_wallet_descriptor(
+        self,
+        secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+        let descriptor = if self.contains('#') {
+            let parts: Vec<&str> = self.splitn(2, '#').collect();
+            if !get_checksum(parts[0])
+                .ok()
+                .map(|computed| computed == parts[1])
+                .unwrap_or(false)
+            {
+                return Err(DescriptorError::InvalidDescriptorChecksum);
+            }
+
+            parts[0]
+        } else {
+            self
+        };
+
+        ExtendedDescriptor::parse_descriptor(secp, descriptor)?
+            .into_wallet_descriptor(secp, network)
+    }
+}
+
+impl IntoWalletDescriptor for &String {
+    fn into_wallet_descriptor(
+        self,
+        secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+        self.as_str().into_wallet_descriptor(secp, network)
+    }
+}
+
+impl IntoWalletDescriptor for ExtendedDescriptor {
+    fn into_wallet_descriptor(
+        self,
+        secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+        (self, KeyMap::default()).into_wallet_descriptor(secp, network)
+    }
+}
+
+impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
+    fn into_wallet_descriptor(
+        self,
+        secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+        use crate::keys::DescriptorKey;
+
+        let check_key = |pk: &DescriptorPublicKey| {
+            let (pk, _, networks) = if self.0.is_witness() {
+                let desciptor_key: DescriptorKey<miniscript::Segwitv0> =
+                    pk.clone().into_descriptor_key()?;
+                desciptor_key.extract(secp)?
+            } else {
+                let desciptor_key: DescriptorKey<miniscript::Legacy> =
+                    pk.clone().into_descriptor_key()?;
+                desciptor_key.extract(secp)?
+            };
+
+            if networks.contains(&network) {
+                Ok(pk)
+            } else {
+                Err(DescriptorError::Key(KeyError::InvalidNetwork))
+            }
+        };
+
+        // check the network for the keys
+        let translated = self.0.translate_pk(check_key, check_key)?;
+
+        Ok((translated, self.1))
+    }
+}
+
+impl IntoWalletDescriptor for DescriptorTemplateOut {
+    fn into_wallet_descriptor(
+        self,
+        _secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+        let valid_networks = &self.2;
+
+        let fix_key = |pk: &DescriptorPublicKey| {
+            if valid_networks.contains(&network) {
+                // workaround for xpubs generated by other key types, like bip39: since when the
+                // conversion is made one network has to be chosen, what we generally choose
+                // "mainnet", but then override the set of valid networks to specify that all of
+                // them are valid. here we reset the network to make sure the wallet struct gets a
+                // descriptor with the right network everywhere.
+                let pk = match pk {
+                    DescriptorPublicKey::XPub(ref xpub) => {
+                        let mut xpub = xpub.clone();
+                        xpub.xkey.network = network;
+
+                        DescriptorPublicKey::XPub(xpub)
+                    }
+                    other => other.clone(),
+                };
+
+                Ok(pk)
+            } else {
+                Err(DescriptorError::Key(KeyError::InvalidNetwork))
+            }
+        };
+
+        // fixup the network for keys that need it
+        let translated = self.0.translate_pk(fix_key, fix_key)?;
+
+        Ok((translated, self.1))
+    }
+}
+
+/// Wrapper for `IntoWalletDescriptor` that performs additional checks on the keys contained in the
+/// descriptor
+pub(crate) fn into_wallet_descriptor_checked<T: IntoWalletDescriptor>(
+    inner: T,
+    secp: &SecpCtx,
+    network: Network,
+) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+    let (descriptor, keymap) = inner.into_wallet_descriptor(secp, network)?;
+
+    // Ensure the keys don't contain any hardened derivation steps or hardened wildcards
+    let descriptor_contains_hardened_steps = descriptor.for_any_key(|k| {
+        if let DescriptorPublicKey::XPub(DescriptorXKey {
+            derivation_path,
+            wildcard,
+            ..
+        }) = k.as_key()
+        {
+            return *wildcard == Wildcard::Hardened
+                || derivation_path.into_iter().any(ChildNumber::is_hardened);
+        }
+
+        false
+    });
+    if descriptor_contains_hardened_steps {
+        return Err(DescriptorError::HardenedDerivationXpub);
+    }
+
+    // Ensure that there are no duplicated keys
+    let mut found_keys = HashSet::new();
+    let descriptor_contains_duplicated_keys = descriptor.for_any_key(|k| {
+        if let DescriptorPublicKey::XPub(xkey) = k.as_key() {
+            let fingerprint = xkey.root_fingerprint(secp);
+            if found_keys.contains(&fingerprint) {
+                return true;
+            }
+
+            found_keys.insert(fingerprint);
+        }
+
+        false
+    });
+    if descriptor_contains_duplicated_keys {
+        return Err(DescriptorError::DuplicatedKeys);
+    }
+
+    Ok((descriptor, keymap))
+}
+
+#[doc(hidden)]
+/// Used internally mainly by the `descriptor!()` and `fragment!()` macros
+pub trait CheckMiniscript<Ctx: miniscript::ScriptContext> {
+    fn check_minsicript(&self) -> Result<(), miniscript::Error>;
+}
+
+impl<Ctx: miniscript::ScriptContext, Pk: miniscript::MiniscriptKey> CheckMiniscript<Ctx>
+    for miniscript::Miniscript<Pk, Ctx>
+{
+    fn check_minsicript(&self) -> Result<(), miniscript::Error> {
+        Ctx::check_global_validity(self)?;
+
+        Ok(())
+    }
+}
 
 /// Trait implemented on [`Descriptor`]s to add a method to extract the spending [`policy`]
 pub trait ExtractPolicy {
+    /// Extract the spending [`policy`]
     fn extract_policy(
         &self,
-        signers: Arc<SignersContainer<DescriptorPublicKey>>,
-    ) -> Result<Option<Policy>, Error>;
+        signers: &SignersContainer,
+        psbt: BuildSatisfaction,
+        secp: &SecpCtx,
+    ) -> Result<Option<Policy>, DescriptorError>;
 }
 
 pub(crate) trait XKeyUtils {
     fn full_path(&self, append: &[ChildNumber]) -> DerivationPath;
-    fn root_fingerprint(&self) -> Fingerprint;
+    fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint;
 }
 
-impl<K: InnerXKey> XKeyUtils for DescriptorXKey<K> {
+// FIXME: `InnerXKey` was made private in rust-miniscript, so we have to implement this manually on
+// both `ExtendedPubKey` and `ExtendedPrivKey`.
+//
+// Revert back to using the trait once https://github.com/rust-bitcoin/rust-miniscript/pull/230 is
+// released
+impl XKeyUtils for DescriptorXKey<ExtendedPubKey> {
     fn full_path(&self, append: &[ChildNumber]) -> DerivationPath {
-        let full_path = match &self.source {
-            &Some((_, ref path)) => path
+        let full_path = match self.origin {
+            Some((_, ref path)) => path
                 .into_iter()
                 .chain(self.derivation_path.into_iter())
                 .cloned()
                 .collect(),
-            &None => self.derivation_path.clone(),
+            None => self.derivation_path.clone(),
         };
 
-        if self.is_wildcard {
+        if self.wildcard != Wildcard::None {
             full_path
                 .into_iter()
-                .chain(append.into_iter())
+                .chain(append.iter())
                 .cloned()
                 .collect()
         } else {
@@ -96,21 +294,61 @@ impl<K: InnerXKey> XKeyUtils for DescriptorXKey<K> {
         }
     }
 
-    fn root_fingerprint(&self) -> Fingerprint {
-        match &self.source {
-            &Some((fingerprint, _)) => fingerprint.clone(),
-            &None => self.xkey.xkey_fingerprint(),
+    fn root_fingerprint(&self, _: &SecpCtx) -> Fingerprint {
+        match self.origin {
+            Some((fingerprint, _)) => fingerprint,
+            None => self.xkey.fingerprint(),
+        }
+    }
+}
+impl XKeyUtils for DescriptorXKey<ExtendedPrivKey> {
+    fn full_path(&self, append: &[ChildNumber]) -> DerivationPath {
+        let full_path = match self.origin {
+            Some((_, ref path)) => path
+                .into_iter()
+                .chain(self.derivation_path.into_iter())
+                .cloned()
+                .collect(),
+            None => self.derivation_path.clone(),
+        };
+
+        if self.wildcard != Wildcard::None {
+            full_path
+                .into_iter()
+                .chain(append.iter())
+                .cloned()
+                .collect()
+        } else {
+            full_path
+        }
+    }
+
+    fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint {
+        match self.origin {
+            Some((fingerprint, _)) => fingerprint,
+            None => self.xkey.fingerprint(secp),
         }
     }
 }
 
-pub(crate) trait DescriptorMeta: Sized {
+pub(crate) trait DerivedDescriptorMeta {
+    fn get_hd_keypaths(&self, secp: &SecpCtx) -> Result<HdKeyPaths, DescriptorError>;
+}
+
+pub(crate) trait DescriptorMeta {
     fn is_witness(&self) -> bool;
-    fn get_hd_keypaths(&self, index: u32) -> Result<HDKeyPaths, Error>;
-    fn is_fixed(&self) -> bool;
-    fn derive_from_hd_keypaths(&self, hd_keypaths: &HDKeyPaths) -> Option<Self>;
-    fn derive_from_psbt_input(&self, psbt_input: &psbt::Input, utxo: Option<TxOut>)
-        -> Option<Self>;
+    fn get_extended_keys(&self) -> Result<Vec<DescriptorXKey<ExtendedPubKey>>, DescriptorError>;
+    fn derive_from_hd_keypaths<'s>(
+        &self,
+        hd_keypaths: &HdKeyPaths,
+        secp: &'s SecpCtx,
+    ) -> Option<DerivedDescriptor<'s>>;
+    fn derive_from_psbt_input<'s>(
+        &self,
+        psbt_input: &psbt::Input,
+        utxo: Option<TxOut>,
+        secp: &'s SecpCtx,
+    ) -> Option<DerivedDescriptor<'s>>;
 }
 
 pub(crate) trait DescriptorScripts {
@@ -118,204 +356,177 @@ pub(crate) trait DescriptorScripts {
     fn psbt_witness_script(&self) -> Option<Script>;
 }
 
-impl<T> DescriptorScripts for Descriptor<T>
-where
-    T: miniscript::MiniscriptKey + miniscript::ToPublicKey,
-{
+impl<'s> DescriptorScripts for DerivedDescriptor<'s> {
     fn psbt_redeem_script(&self) -> Option<Script> {
-        match self {
-            Descriptor::ShWpkh(_) => Some(self.witness_script()),
-            Descriptor::ShWsh(ref script) => Some(script.encode().to_v0_p2wsh()),
-            Descriptor::Sh(ref script) => Some(script.encode()),
-            Descriptor::Bare(ref script) => Some(script.encode()),
+        match self.desc_type() {
+            DescriptorType::ShWpkh => Some(self.explicit_script()),
+            DescriptorType::ShWsh => Some(self.explicit_script().to_v0_p2wsh()),
+            DescriptorType::Sh => Some(self.explicit_script()),
+            DescriptorType::Bare => Some(self.explicit_script()),
+            DescriptorType::ShSortedMulti => Some(self.explicit_script()),
             _ => None,
         }
     }
 
     fn psbt_witness_script(&self) -> Option<Script> {
-        match self {
-            Descriptor::Wsh(ref script) => Some(script.encode()),
-            Descriptor::ShWsh(ref script) => Some(script.encode()),
+        match self.desc_type() {
+            DescriptorType::Wsh => Some(self.explicit_script()),
+            DescriptorType::ShWsh => Some(self.explicit_script()),
+            DescriptorType::WshSortedMulti | DescriptorType::ShWshSortedMulti => {
+                Some(self.explicit_script())
+            }
             _ => None,
         }
     }
 }
 
-impl DescriptorMeta for Descriptor<DescriptorPublicKey> {
+impl DescriptorMeta for ExtendedDescriptor {
     fn is_witness(&self) -> bool {
-        match self {
-            Descriptor::Bare(_) | Descriptor::Pk(_) | Descriptor::Pkh(_) | Descriptor::Sh(_) => {
-                false
-            }
-            Descriptor::Wpkh(_)
-            | Descriptor::ShWpkh(_)
-            | Descriptor::Wsh(_)
-            | Descriptor::ShWsh(_) => true,
-        }
+        matches!(
+            self.desc_type(),
+            DescriptorType::Wpkh
+                | DescriptorType::ShWpkh
+                | DescriptorType::Wsh
+                | DescriptorType::ShWsh
+                | DescriptorType::ShWshSortedMulti
+                | DescriptorType::WshSortedMulti
+        )
     }
 
-    fn get_hd_keypaths(&self, index: u32) -> Result<HDKeyPaths, Error> {
-        let mut answer = BTreeMap::new();
+    fn get_extended_keys(&self) -> Result<Vec<DescriptorXKey<ExtendedPubKey>>, DescriptorError> {
+        let mut answer = Vec::new();
 
-        let translatefpk = |key: &DescriptorPublicKey| -> Result<_, Error> {
-            match key {
-                DescriptorPublicKey::PubKey(_) => {}
-                DescriptorPublicKey::XPub(xpub) => {
-                    let derive_path = if xpub.is_wildcard {
-                        xpub.derivation_path
-                            .into_iter()
-                            .chain([ChildNumber::from_normal_idx(index)?].iter())
-                            .cloned()
-                            .collect()
-                    } else {
-                        xpub.derivation_path.clone()
-                    };
-                    let derived_pubkey = xpub
-                        .xkey
-                        .derive_pub(&Secp256k1::verification_only(), &derive_path)?;
-
-                    answer.insert(
-                        derived_pubkey.public_key,
-                        (
-                            xpub.root_fingerprint(),
-                            xpub.full_path(&[ChildNumber::from_normal_idx(index)?]),
-                        ),
-                    );
-                }
+        self.for_each_key(|pk| {
+            if let DescriptorPublicKey::XPub(xpub) = pk.as_key() {
+                answer.push(xpub.clone());
             }
 
-            Ok(DummyKey::default())
-        };
-        let translatefpkh = |_: &hash160::Hash| -> Result<_, Error> { Ok(DummyKey::default()) };
-
-        self.translate_pk(translatefpk, translatefpkh)?;
+            true
+        });
 
         Ok(answer)
     }
 
-    fn is_fixed(&self) -> bool {
-        let mut found_wildcard = false;
+    fn derive_from_hd_keypaths<'s>(
+        &self,
+        hd_keypaths: &HdKeyPaths,
+        secp: &'s SecpCtx,
+    ) -> Option<DerivedDescriptor<'s>> {
+        let index: HashMap<_, _> = hd_keypaths.values().map(|(a, b)| (a, b)).collect();
 
-        let translatefpk = |key: &DescriptorPublicKey| -> Result<_, Error> {
-            match key {
-                DescriptorPublicKey::PubKey(_) => {}
-                DescriptorPublicKey::XPub(xpub) => {
-                    if xpub.is_wildcard {
-                        found_wildcard = true;
-                    }
-                }
-            }
-
-            Ok(DummyKey::default())
-        };
-        let translatefpkh = |_: &hash160::Hash| -> Result<_, Error> { Ok(DummyKey::default()) };
-
-        self.translate_pk(translatefpk, translatefpkh).unwrap();
-
-        !found_wildcard
-    }
-
-    fn derive_from_hd_keypaths(&self, hd_keypaths: &HDKeyPaths) -> Option<Self> {
-        let index: HashMap<_, _> = hd_keypaths.values().cloned().collect();
-
-        let mut derive_path = None::<DerivationPath>;
-        let translatefpk = |key: &DescriptorPublicKey| -> Result<_, Error> {
-            if derive_path.is_some() {
+        let mut path_found = None;
+        self.for_each_key(|key| {
+            if path_found.is_some() {
                 // already found a matching path, we are done
-                return Ok(DummyKey::default());
+                return true;
             }
 
-            if let DescriptorPublicKey::XPub(xpub) = key {
+            if let DescriptorPublicKey::XPub(xpub) = key.as_key().deref() {
                 // Check if the key matches one entry in our `index`. If it does, `matches()` will
                 // return the "prefix" that matched, so we remove that prefix from the full path
-                // found in `index` and save it in `derive_path`
-                let root_fingerprint = xpub.root_fingerprint();
-                derive_path = index
+                // found in `index` and save it in `derive_path`. We expect this to be a derivation
+                // path of length 1 if the key is `wildcard` and an empty path otherwise.
+                let root_fingerprint = xpub.root_fingerprint(secp);
+                let derivation_path: Option<Vec<ChildNumber>> = index
                     .get_key_value(&root_fingerprint)
-                    .and_then(|(fingerprint, path)| xpub.matches(*fingerprint, path))
-                    .map(|prefix_path| prefix_path.into_iter().cloned().collect::<Vec<_>>())
+                    .and_then(|(fingerprint, path)| {
+                        xpub.matches(&(**fingerprint, (*path).clone()), secp)
+                    })
                     .map(|prefix| {
                         index
-                            .get(&xpub.root_fingerprint())
+                            .get(&xpub.root_fingerprint(secp))
                             .unwrap()
                             .into_iter()
-                            .skip(prefix.len())
+                            .skip(prefix.into_iter().count())
                             .cloned()
                             .collect()
                     });
+
+                match derivation_path {
+                    Some(path) if xpub.wildcard != Wildcard::None && path.len() == 1 => {
+                        // Ignore hardened wildcards
+                        if let ChildNumber::Normal { index } = path[0] {
+                            path_found = Some(index)
+                        }
+                    }
+                    Some(path) if xpub.wildcard == Wildcard::None && path.is_empty() => {
+                        path_found = Some(0)
+                    }
+                    _ => {}
+                }
             }
 
-            Ok(DummyKey::default())
-        };
-        let translatefpkh = |_: &hash160::Hash| -> Result<_, Error> { Ok(DummyKey::default()) };
+            true
+        });
 
-        self.translate_pk(translatefpk, translatefpkh).unwrap();
-
-        derive_path.map(|path| self.derive(path.as_ref()))
+        path_found.map(|path| self.as_derived(path, secp))
     }
 
-    fn derive_from_psbt_input(
+    fn derive_from_psbt_input<'s>(
         &self,
         psbt_input: &psbt::Input,
         utxo: Option<TxOut>,
-    ) -> Option<Self> {
-        if let Some(derived) = self.derive_from_hd_keypaths(&psbt_input.hd_keypaths) {
+        secp: &'s SecpCtx,
+    ) -> Option<DerivedDescriptor<'s>> {
+        if let Some(derived) = self.derive_from_hd_keypaths(&psbt_input.bip32_derivation, secp) {
             return Some(derived);
-        } else if !self.is_fixed() {
-            // If the descriptor is not fixed we can't brute-force the derivation address, so just
-            // exit here
+        }
+        if self.is_deriveable() {
+            // We can't try to bruteforce the derivation index, exit here
             return None;
         }
 
-        match self {
-            Descriptor::Pk(_)
-            | Descriptor::Pkh(_)
-            | Descriptor::Wpkh(_)
-            | Descriptor::ShWpkh(_)
+        let descriptor = self.as_derived_fixed(secp);
+        match descriptor.desc_type() {
+            // TODO: add pk() here
+            DescriptorType::Pkh | DescriptorType::Wpkh | DescriptorType::ShWpkh
                 if utxo.is_some()
-                    && self.script_pubkey() == utxo.as_ref().unwrap().script_pubkey =>
+                    && descriptor.script_pubkey() == utxo.as_ref().unwrap().script_pubkey =>
             {
-                Some(self.clone())
+                Some(descriptor)
             }
-            Descriptor::Bare(ms) | Descriptor::Sh(ms)
+            DescriptorType::Bare | DescriptorType::Sh | DescriptorType::ShSortedMulti
                 if psbt_input.redeem_script.is_some()
-                    && &ms.encode() == psbt_input.redeem_script.as_ref().unwrap() =>
+                    && &descriptor.explicit_script()
+                        == psbt_input.redeem_script.as_ref().unwrap() =>
             {
-                Some(self.clone())
+                Some(descriptor)
             }
-            Descriptor::Wsh(ms) | Descriptor::ShWsh(ms)
+            DescriptorType::Wsh
+            | DescriptorType::ShWsh
+            | DescriptorType::ShWshSortedMulti
+            | DescriptorType::WshSortedMulti
                 if psbt_input.witness_script.is_some()
-                    && &ms.encode() == psbt_input.witness_script.as_ref().unwrap() =>
+                    && &descriptor.explicit_script()
+                        == psbt_input.witness_script.as_ref().unwrap() =>
             {
-                Some(self.clone())
+                Some(descriptor)
             }
             _ => None,
         }
     }
 }
 
-#[derive(Debug, Clone, Hash, PartialEq, PartialOrd, Eq, Ord, Default)]
-struct DummyKey();
+impl<'s> DerivedDescriptorMeta for DerivedDescriptor<'s> {
+    fn get_hd_keypaths(&self, secp: &SecpCtx) -> Result<HdKeyPaths, DescriptorError> {
+        let mut answer = BTreeMap::new();
+        self.for_each_key(|key| {
+            if let DescriptorPublicKey::XPub(xpub) = key.as_key().deref() {
+                let derived_pubkey = xpub
+                    .xkey
+                    .derive_pub(secp, &xpub.derivation_path)
+                    .expect("Derivation can't fail");
 
-impl fmt::Display for DummyKey {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "DummyKey")
-    }
-}
+                answer.insert(
+                    derived_pubkey.public_key,
+                    (xpub.root_fingerprint(secp), xpub.full_path(&[])),
+                );
+            }
 
-impl std::str::FromStr for DummyKey {
-    type Err = ();
+            true
+        });
 
-    fn from_str(_: &str) -> Result<Self, Self::Err> {
-        Ok(DummyKey::default())
-    }
-}
-
-impl miniscript::MiniscriptKey for DummyKey {
-    type Hash = DummyKey;
-
-    fn to_pubkeyhash(&self) -> DummyKey {
-        DummyKey::default()
+        Ok(answer)
     }
 }
 
@@ -325,10 +536,11 @@ mod test {
 
     use bitcoin::consensus::encode::deserialize;
     use bitcoin::hashes::hex::FromHex;
-    use bitcoin::util::psbt;
+    use bitcoin::secp256k1::Secp256k1;
+    use bitcoin::util::{bip32, psbt};
 
     use super::*;
-    use crate::psbt::PSBTUtils;
+    use crate::psbt::PsbtUtils;
 
     #[test]
     fn test_derive_from_psbt_input_wpkh_wif() {
@@ -349,7 +561,7 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
 
@@ -380,7 +592,7 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
 
@@ -404,7 +616,7 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
 
@@ -434,7 +646,159 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
+
+    #[test]
+    fn test_to_wallet_descriptor_fixup_networks() {
+        use crate::keys::{any_network, IntoDescriptorKey};
+
+        let secp = Secp256k1::new();
+
+        let xpub = bip32::ExtendedPubKey::from_str("xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL").unwrap();
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+
+        // here `to_descriptor_key` will set the valid networks for the key to only mainnet, since
+        // we are using an "xpub"
+        let key = (xpub, path).into_descriptor_key().unwrap();
+        // override it with any. this happens in some key conversions, like bip39
+        let key = key.override_valid_networks(any_network());
+
+        // make a descriptor out of it
+        let desc = crate::descriptor!(wpkh(key)).unwrap();
+        // this should conver the key that supports "any_network" to the right network (testnet)
+        let (wallet_desc, _) = desc
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .unwrap();
+
+        assert_eq!(wallet_desc.to_string(), "wpkh(tpubDEnoLuPdBep9bzw5LoGYpsxUQYheRQ9gcgrJhJEcdKFB9cWQRyYmkCyRoTqeD4tJYiVVgt6A3rN6rWn9RYhR9sBsGxji29LYWHuKKbdb1ev/0/*)#y8p7e8kk");
+    }
+
+    // test IntoWalletDescriptor trait from &str with and without checksum appended
+    #[test]
+    fn test_descriptor_from_str_with_checksum() {
+        let secp = Secp256k1::new();
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#tqz0nc62"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)#67ju93jw"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#67ju93jw"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(matches!(
+            desc.err(),
+            Some(DescriptorError::InvalidDescriptorChecksum)
+        ));
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#67ju93jw"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(matches!(
+            desc.err(),
+            Some(DescriptorError::InvalidDescriptorChecksum)
+        ));
+    }
+
+    // test IntoWalletDescriptor trait from &str with keys from right and wrong network
+    #[test]
+    fn test_descriptor_from_str_with_keys_network() {
+        let secp = Secp256k1::new();
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Regtest);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Regtest);
+        assert!(desc.is_ok());
+
+        let desc = "sh(wpkh(02864bb4ad00cefa806098a69e192bbda937494e69eb452b87bb3f20f6283baedb))"
+            .into_wallet_descriptor(&secp, Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "sh(wpkh(02864bb4ad00cefa806098a69e192bbda937494e69eb452b87bb3f20f6283baedb))"
+            .into_wallet_descriptor(&secp, Network::Bitcoin);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Bitcoin);
+        assert!(matches!(
+            desc.err(),
+            Some(DescriptorError::Key(KeyError::InvalidNetwork))
+        ));
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .into_wallet_descriptor(&secp, Network::Bitcoin);
+        assert!(matches!(
+            desc.err(),
+            Some(DescriptorError::Key(KeyError::InvalidNetwork))
+        ));
+    }
+
+    // test IntoWalletDescriptor trait from the output of the descriptor!() macro
+    #[test]
+    fn test_descriptor_from_str_from_output_of_macro() {
+        let secp = Secp256k1::new();
+
+        let tpub = bip32::ExtendedPubKey::from_str("tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK").unwrap();
+        let path = bip32::DerivationPath::from_str("m/1/2").unwrap();
+        let key = (tpub, path).into_descriptor_key().unwrap();
+
+        // make a descriptor out of it
+        let desc = crate::descriptor!(wpkh(key)).unwrap();
+
+        let (wallet_desc, _) = desc
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .unwrap();
+        let wallet_desc_str = wallet_desc.to_string();
+        assert_eq!(wallet_desc_str, "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)#67ju93jw");
+
+        let (wallet_desc2, _) = wallet_desc_str
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .unwrap();
+        assert_eq!(wallet_desc, wallet_desc2)
+    }
+
+    #[test]
+    fn test_into_wallet_descriptor_checked() {
+        let secp = Secp256k1::new();
+
+        let descriptor = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0'/1/2/*)";
+        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+
+        assert!(result.is_err());
+        assert!(matches!(
+            result.unwrap_err(),
+            DescriptorError::HardenedDerivationXpub
+        ));
+
+        let descriptor = "wsh(multi(2,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/*))";
+        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+
+        assert!(result.is_err());
+        assert!(matches!(
+            result.unwrap_err(),
+            DescriptorError::DuplicatedKeys
+        ));
+    }
 }

src/descriptor/template.rs

@@ -0,0 +1,726 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Descriptor templates
+//!
+//! This module contains the definition of various common script templates that are ready to be
+//! used. See the documentation of each template for an example.
+
+use bitcoin::util::bip32;
+use bitcoin::Network;
+
+use miniscript::{Legacy, Segwitv0};
+
+use super::{ExtendedDescriptor, IntoWalletDescriptor, KeyMap};
+use crate::descriptor::DescriptorError;
+use crate::keys::{DerivableKey, IntoDescriptorKey, ValidNetworks};
+use crate::wallet::utils::SecpCtx;
+use crate::{descriptor, KeychainKind};
+
+/// Type alias for the return type of [`DescriptorTemplate`], [`descriptor!`](crate::descriptor!) and others
+pub type DescriptorTemplateOut = (ExtendedDescriptor, KeyMap, ValidNetworks);
+
+/// Trait for descriptor templates that can be built into a full descriptor
+///
+/// Since [`IntoWalletDescriptor`] is implemented for any [`DescriptorTemplate`], they can also be
+/// passed directly to the [`Wallet`](crate::Wallet) constructor.
+///
+/// ## Example
+///
+/// ```
+/// use bdk::descriptor::error::Error as DescriptorError;
+/// use bdk::keys::{IntoDescriptorKey, KeyError};
+/// use bdk::miniscript::Legacy;
+/// use bdk::template::{DescriptorTemplate, DescriptorTemplateOut};
+///
+/// struct MyP2PKH<K: IntoDescriptorKey<Legacy>>(K);
+///
+/// impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for MyP2PKH<K> {
+///     fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+///         Ok(bdk::descriptor!(pkh(self.0))?)
+///     }
+/// }
+/// ```
+pub trait DescriptorTemplate {
+    /// Build the complete descriptor
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError>;
+}
+
+/// Turns a [`DescriptorTemplate`] into a valid wallet descriptor by calling its
+/// [`build`](DescriptorTemplate::build) method
+impl<T: DescriptorTemplate> IntoWalletDescriptor for T {
+    fn into_wallet_descriptor(
+        self,
+        secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+        self.build()?.into_wallet_descriptor(secp, network)
+    }
+}
+
+/// P2PKH template. Expands to a descriptor `pkh(key)`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::P2Pkh;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet = Wallet::new_offline(
+///     P2Pkh(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_address(New)?.to_string(),
+///     "mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct P2Pkh<K: IntoDescriptorKey<Legacy>>(pub K);
+
+impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for P2Pkh<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        descriptor!(pkh(self.0))
+    }
+}
+
+/// P2WPKH-P2SH template. Expands to a descriptor `sh(wpkh(key))`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::P2Wpkh_P2Sh;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet = Wallet::new_offline(
+///     P2Wpkh_P2Sh(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_address(New)?.to_string(),
+///     "2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+#[allow(non_camel_case_types)]
+pub struct P2Wpkh_P2Sh<K: IntoDescriptorKey<Segwitv0>>(pub K);
+
+impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh_P2Sh<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        descriptor!(sh(wpkh(self.0)))
+    }
+}
+
+/// P2WPKH template. Expands to a descriptor `wpkh(key)`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::P2Wpkh;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet = Wallet::new_offline(
+///     P2Wpkh(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_address(New)?.to_string(),
+///     "tb1q4525hmgw265tl3drrl8jjta7ayffu6jf68ltjd"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct P2Wpkh<K: IntoDescriptorKey<Segwitv0>>(pub K);
+
+impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        descriptor!(wpkh(self.0))
+    }
+}
+
+/// BIP44 template. Expands to `pkh(key/44'/0'/0'/{0,1}/*)`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`Bip44Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet,  KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip44;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet = Wallet::new_offline(
+///     Bip44(key.clone(), KeychainKind::External),
+///     Some(Bip44(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_address(New)?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#xgaaevjx");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct Bip44<K: DerivableKey<Legacy>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        P2Pkh(legacy::make_bipxx_private(44, self.0, self.1)?).build()
+    }
+}
+
+/// BIP44 public template. Expands to `pkh(key/{0,1}/*)`
+///
+/// This assumes that the key used has already been derived with `m/44'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`Bip44`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet,  KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip44Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet = Wallet::new_offline(
+///     Bip44Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(Bip44Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_address(New)?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#xgaaevjx");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct Bip44Public<K: DerivableKey<Legacy>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        P2Pkh(legacy::make_bipxx_public(44, self.0, self.1, self.2)?).build()
+    }
+}
+
+/// BIP49 template. Expands to `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`Bip49Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet,  KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip49;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet = Wallet::new_offline(
+///     Bip49(key.clone(), KeychainKind::External),
+///     Some(Bip49(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_address(New)?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#gsmdv4xr");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct Bip49<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        P2Wpkh_P2Sh(segwit_v0::make_bipxx_private(49, self.0, self.1)?).build()
+    }
+}
+
+/// BIP49 public template. Expands to `sh(wpkh(key/{0,1}/*))`
+///
+/// This assumes that the key used has already been derived with `m/49'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`Bip49`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet,  KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip49Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet = Wallet::new_offline(
+///     Bip49Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(Bip49Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_address(New)?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#gsmdv4xr");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct Bip49Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        P2Wpkh_P2Sh(segwit_v0::make_bipxx_public(49, self.0, self.1, self.2)?).build()
+    }
+}
+
+/// BIP84 template. Expands to `wpkh(key/84'/0'/0'/{0,1}/*)`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`Bip84Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet,  KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip84;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet = Wallet::new_offline(
+///     Bip84(key.clone(), KeychainKind::External),
+///     Some(Bip84(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_address(New)?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#nkk5dtkg");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct Bip84<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        P2Wpkh(segwit_v0::make_bipxx_private(84, self.0, self.1)?).build()
+    }
+}
+
+/// BIP84 public template. Expands to `wpkh(key/{0,1}/*)`
+///
+/// This assumes that the key used has already been derived with `m/84'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`Bip84`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet,  KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// # use bdk::wallet::AddressIndex::New;
+/// use bdk::template::Bip84Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet = Wallet::new_offline(
+///     Bip84Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(Bip84Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_address(New)?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#nkk5dtkg");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct Bip84Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, DescriptorError> {
+        P2Wpkh(segwit_v0::make_bipxx_public(84, self.0, self.1, self.2)?).build()
+    }
+}
+
+macro_rules! expand_make_bipxx {
+    ( $mod_name:ident, $ctx:ty ) => {
+        mod $mod_name {
+            use super::*;
+
+            pub(super) fn make_bipxx_private<K: DerivableKey<$ctx>>(
+                bip: u32,
+                key: K,
+                keychain: KeychainKind,
+            ) -> Result<impl IntoDescriptorKey<$ctx>, DescriptorError> {
+                let mut derivation_path = Vec::with_capacity(4);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(bip)?);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+
+                match keychain {
+                    KeychainKind::External => {
+                        derivation_path.push(bip32::ChildNumber::from_normal_idx(0)?)
+                    }
+                    KeychainKind::Internal => {
+                        derivation_path.push(bip32::ChildNumber::from_normal_idx(1)?)
+                    }
+                };
+
+                let derivation_path: bip32::DerivationPath = derivation_path.into();
+
+                Ok((key, derivation_path))
+            }
+            pub(super) fn make_bipxx_public<K: DerivableKey<$ctx>>(
+                bip: u32,
+                key: K,
+                parent_fingerprint: bip32::Fingerprint,
+                keychain: KeychainKind,
+            ) -> Result<impl IntoDescriptorKey<$ctx>, DescriptorError> {
+                let derivation_path: bip32::DerivationPath = match keychain {
+                    KeychainKind::External => vec![bip32::ChildNumber::from_normal_idx(0)?].into(),
+                    KeychainKind::Internal => vec![bip32::ChildNumber::from_normal_idx(1)?].into(),
+                };
+
+                let source_path = bip32::DerivationPath::from(vec![
+                    bip32::ChildNumber::from_hardened_idx(bip)?,
+                    bip32::ChildNumber::from_hardened_idx(0)?,
+                    bip32::ChildNumber::from_hardened_idx(0)?,
+                ]);
+
+                Ok((key, (parent_fingerprint, source_path), derivation_path))
+            }
+        }
+    };
+}
+
+expand_make_bipxx!(legacy, Legacy);
+expand_make_bipxx!(segwit_v0, Segwitv0);
+
+#[cfg(test)]
+mod test {
+    // test existing descriptor templates, make sure they are expanded to the right descriptors
+
+    use std::str::FromStr;
+
+    use super::*;
+    use crate::descriptor::derived::AsDerived;
+    use crate::descriptor::{DescriptorError, DescriptorMeta};
+    use crate::keys::ValidNetworks;
+    use bitcoin::network::constants::Network::Regtest;
+    use bitcoin::secp256k1::Secp256k1;
+    use miniscript::descriptor::{DescriptorPublicKey, DescriptorTrait, KeyMap};
+    use miniscript::Descriptor;
+
+    // verify template descriptor generates expected address(es)
+    fn check(
+        desc: Result<(Descriptor<DescriptorPublicKey>, KeyMap, ValidNetworks), DescriptorError>,
+        is_witness: bool,
+        is_fixed: bool,
+        expected: &[&str],
+    ) {
+        let secp = Secp256k1::new();
+
+        let (desc, _key_map, _networks) = desc.unwrap();
+        assert_eq!(desc.is_witness(), is_witness);
+        assert_eq!(!desc.is_deriveable(), is_fixed);
+        for i in 0..expected.len() {
+            let index = i as u32;
+            let child_desc = if !desc.is_deriveable() {
+                desc.as_derived_fixed(&secp)
+            } else {
+                desc.as_derived(index, &secp)
+            };
+            let address = child_desc.address(Regtest).unwrap();
+            assert_eq!(address.to_string(), *expected.get(i).unwrap());
+        }
+    }
+
+    // P2PKH
+    #[test]
+    fn test_p2ph_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2Pkh(prvkey).build(),
+            false,
+            true,
+            &["mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2Pkh(pubkey).build(),
+            false,
+            true,
+            &["muZpTpBYhxmRFuCjLc7C6BBDF32C8XVJUi"],
+        );
+    }
+
+    // P2WPKH-P2SH `sh(wpkh(key))`
+    #[test]
+    fn test_p2wphp2sh_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2Wpkh_P2Sh(prvkey).build(),
+            true,
+            true,
+            &["2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2Wpkh_P2Sh(pubkey).build(),
+            true,
+            true,
+            &["2N5LiC3CqzxDamRTPG1kiNv1FpNJQ7x28sb"],
+        );
+    }
+
+    // P2WPKH `wpkh(key)`
+    #[test]
+    fn test_p2wph_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2Wpkh(prvkey).build(),
+            true,
+            true,
+            &["bcrt1q4525hmgw265tl3drrl8jjta7ayffu6jfcwxx9y"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2Wpkh(pubkey).build(),
+            true,
+            true,
+            &["bcrt1qngw83fg8dz0k749cg7k3emc7v98wy0c7azaa6h"],
+        );
+    }
+
+    // BIP44 `pkh(key/44'/0'/0'/{0,1}/*)`
+    #[test]
+    fn test_bip44_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            Bip44(prvkey, KeychainKind::External).build(),
+            false,
+            false,
+            &[
+                "n453VtnjDHPyDt2fDstKSu7A3YCJoHZ5g5",
+                "mvfrrumXgTtwFPWDNUecBBgzuMXhYM7KRP",
+                "mzYvhRAuQqbdSKMVVzXNYyqihgNdRadAUQ",
+            ],
+        );
+        check(
+            Bip44(prvkey, KeychainKind::Internal).build(),
+            false,
+            false,
+            &[
+                "muHF98X9KxEzdKrnFAX85KeHv96eXopaip",
+                "n4hpyLJE5ub6B5Bymv4eqFxS5KjrewSmYR",
+                "mgvkdv1ffmsXd2B1sRKQ5dByK3SzpG42rA",
+            ],
+        );
+    }
+
+    // BIP44 public `pkh(key/{0,1}/*)`
+    #[test]
+    fn test_bip44_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            Bip44Public(pubkey, fingerprint, KeychainKind::External).build(),
+            false,
+            false,
+            &[
+                "miNG7dJTzJqNbFS19svRdTCisC65dsubtR",
+                "n2UqaDbCjWSFJvpC84m3FjUk5UaeibCzYg",
+                "muCPpS6Ue7nkzeJMWDViw7Lkwr92Yc4K8g",
+            ],
+        );
+        check(
+            Bip44Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            false,
+            false,
+            &[
+                "moDr3vJ8wpt5nNxSK55MPq797nXJb2Ru9H",
+                "ms7A1Yt4uTezT2XkefW12AvLoko8WfNJMG",
+                "mhYiyat2rtEnV77cFfQsW32y1m2ceCGHPo",
+            ],
+        );
+    }
+
+    // BIP49 `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
+    #[test]
+    fn test_bip49_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            Bip49(prvkey, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "2N9bCAJXGm168MjVwpkBdNt6ucka3PKVoUV",
+                "2NDckYkqrYyDMtttEav5hB3Bfw9EGAW5HtS",
+                "2NAFTVtksF9T4a97M7nyCjwUBD24QevZ5Z4",
+            ],
+        );
+        check(
+            Bip49(prvkey, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "2NB3pA8PnzJLGV8YEKNDFpbViZv3Bm1K6CG",
+                "2NBiX2Wzxngb5rPiWpUiJQ2uLVB4HBjFD4p",
+                "2NA8ek4CdQ6aMkveYF6AYuEYNrftB47QGTn",
+            ],
+        );
+    }
+
+    // BIP49 public `sh(wpkh(key/{0,1}/*))`
+    #[test]
+    fn test_bip49_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            Bip49Public(pubkey, fingerprint, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt",
+                "2NCTQfJ1sZa3wQ3pPseYRHbaNEpC3AquEfX",
+                "2MveFxAuC8BYPzTybx7FxSzW8HSd8ATT4z7",
+            ],
+        );
+        check(
+            Bip49Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "2NF2vttKibwyxigxtx95Zw8K7JhDbo5zPVJ",
+                "2Mtmyd8taksxNVWCJ4wVvaiss7QPZGcAJuH",
+                "2NBs3CTVYPr1HCzjB4YFsnWCPCtNg8uMEfp",
+            ],
+        );
+    }
+
+    // BIP84 `wpkh(key/84'/0'/0'/{0,1}/*)`
+    #[test]
+    fn test_bip84_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            Bip84(prvkey, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "bcrt1qkmvk2nadgplmd57ztld8nf8v2yxkzmdvwtjf8s",
+                "bcrt1qx0v6zgfwe50m4kqc58cqzcyem7ay2sfl3gvqhp",
+                "bcrt1q4h7fq9zhxst6e69p3n882nfj649l7w9g3zccfp",
+            ],
+        );
+        check(
+            Bip84(prvkey, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "bcrt1qtrwtz00wxl69e5xex7amy4xzlxkaefg3gfdkxa",
+                "bcrt1qqqasfhxpkkf7zrxqnkr2sfhn74dgsrc3e3ky45",
+                "bcrt1qpks7n0gq74hsgsz3phn5vuazjjq0f5eqhsgyce",
+            ],
+        );
+    }
+
+    // BIP84 public `wpkh(key/{0,1}/*)`
+    #[test]
+    fn test_bip84_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            Bip84Public(pubkey, fingerprint, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "bcrt1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2prcdvd0h",
+                "bcrt1q3lncdlwq3lgcaaeyruynjnlccr0ve0kakh6ana",
+                "bcrt1qt9800y6xl3922jy3uyl0z33jh5wfpycyhcylr9",
+            ],
+        );
+        check(
+            Bip84Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "bcrt1qm6wqukenh7guu792lj2njgw9n78cmwsy8xy3z2",
+                "bcrt1q694twxtjn4nnrvnyvra769j0a23rllj5c6cgwp",
+                "bcrt1qhlac3c5ranv5w5emlnqs7wxhkxt8maelylcarp",
+            ],
+        );
+    }
+}

src/doctest.rs

@@ -1,3 +1,14 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
 #[doc(include = "../README.md")]
 #[cfg(doctest)]
 pub struct ReadmeDoctests;

src/error.rs

@@ -1,88 +1,149 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::fmt;
 
-use bitcoin::{Address, OutPoint};
+use crate::bitcoin::Network;
+use crate::{descriptor, wallet, wallet::address_validator};
+use bitcoin::OutPoint;
 
 /// Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet)
 #[derive(Debug)]
 pub enum Error {
-    KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
-    MissingInputUTXO(usize),
+    /// Wrong number of bytes found when trying to convert to u32
     InvalidU32Bytes(Vec<u8>),
+    /// Generic error
     Generic(String),
+    /// This error is thrown when trying to convert Bare and Public key script to address
     ScriptDoesntHaveAddressForm,
-    SendAllMultipleOutputs,
-    NoAddressees,
+    /// Found multiple outputs when `single_recipient` option has been specified
+    SingleRecipientMultipleOutputs,
+    /// `single_recipient` option is selected but neither `drain_wallet` nor `manually_selected_only` are
+    SingleRecipientNoInputs,
+    /// Cannot build a tx without recipients
+    NoRecipients,
+    /// `manually_selected_only` option is selected but no utxo has been passed
+    NoUtxosSelected,
+    /// Output created is under the dust limit, 546 satoshis
     OutputBelowDustLimit(usize),
-    InsufficientFunds,
-    InvalidAddressNetwork(Address),
-    UnknownUTXO,
-    DifferentTransactions,
+    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
+    InsufficientFunds {
+        /// Sats needed for some transaction
+        needed: u64,
+        /// Sats available for spending
+        available: u64,
+    },
+    /// Branch and bound coin selection possible attempts with sufficiently big UTXO set could grow
+    /// exponentially, thus a limit is set, and when hit, this error is thrown
+    BnBTotalTriesExceeded,
+    /// Branch and bound coin selection tries to avoid needing a change by finding the right inputs for
+    /// the desired outputs plus fee, if there is not such combination this error is thrown
+    BnBNoExactMatch,
+    /// Happens when trying to spend an UTXO that is not in the internal database
+    UnknownUtxo,
+    /// Thrown when a tx is not found in the internal database
     TransactionNotFound,
+    /// Happens when trying to bump a transaction that is already confirmed
     TransactionConfirmed,
+    /// Trying to replace a tx that has a sequence >= `0xFFFFFFFE`
     IrreplaceableTransaction,
+    /// When bumping a tx the fee rate requested is lower than required
     FeeRateTooLow {
+        /// Required fee rate (satoshi/vbyte)
         required: crate::types::FeeRate,
     },
-
+    /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
+    FeeTooLow {
+        /// Required fee absolute value (satoshi)
+        required: u64,
+    },
+    /// Node doesn't have data to estimate a fee rate
+    FeeRateUnavailable,
+    /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
+    /// key in the descriptor must either be a master key itself (having depth = 0) or have an
+    /// explicit origin provided
+    ///
+    /// [`TxBuilder::add_global_xpubs`]: crate::wallet::tx_builder::TxBuilder::add_global_xpubs
+    MissingKeyOrigin(String),
+    /// Error while working with [`keys`](crate::keys)
+    Key(crate::keys::KeyError),
+    /// Descriptor checksum mismatch
     ChecksumMismatch,
-    DifferentDescriptorStructure,
-
-    SpendingPolicyRequired,
+    /// Spending policy is not compatible with this [`KeychainKind`](crate::types::KeychainKind)
+    SpendingPolicyRequired(crate::types::KeychainKind),
+    /// Error while extracting and manipulating policies
     InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
-
+    /// Signing error
     Signer(crate::wallet::signer::SignerError),
+    /// Invalid network
+    InvalidNetwork {
+        /// requested network, for example what is given as bdk-cli option
+        requested: Network,
+        /// found network, for example the network of the bitcoin node
+        found: Network,
+    },
+    #[cfg(feature = "verify")]
+    /// Transaction verification error
+    Verification(crate::wallet::verify::VerifyError),
 
-    // Blockchain interface errors
-    Uncapable(crate::blockchain::Capability),
-    OfflineClient,
+    /// Progress value must be between `0.0` (included) and `100.0` (included)
     InvalidProgressValue(f32),
+    /// Progress update error (maybe the channel has been closed)
     ProgressUpdateError,
-    MissingCachedAddresses,
+    /// Requested outpoint doesn't exist in the tx (vout greater than available outputs)
     InvalidOutpoint(OutPoint),
 
+    /// Error related to the parsing and usage of descriptors
     Descriptor(crate::descriptor::error::Error),
+    /// Error that can be returned to fail the validation of an address
     AddressValidator(crate::wallet::address_validator::AddressValidatorError),
-
+    /// Encoding error
     Encode(bitcoin::consensus::encode::Error),
+    /// Miniscript error
     Miniscript(miniscript::Error),
-    BIP32(bitcoin::util::bip32::Error),
+    /// BIP32 error
+    Bip32(bitcoin::util::bip32::Error),
+    /// An ECDSA error
     Secp256k1(bitcoin::secp256k1::Error),
-    JSON(serde_json::Error),
+    /// Error serializing or deserializing JSON data
+    Json(serde_json::Error),
+    /// Hex decoding error
     Hex(bitcoin::hashes::hex::Error),
-    PSBT(bitcoin::util::psbt::Error),
+    /// Partially signed bitcoin transaction error
+    Psbt(bitcoin::util::psbt::Error),
+    /// Partially signed bitcoin transaction parseerror
+    PsbtParse(bitcoin::util::psbt::PsbtParseError),
 
+    //KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
+    //MissingInputUTXO(usize),
+    //InvalidAddressNetwork(Address),
+    //DifferentTransactions,
+    //DifferentDescriptorStructure,
+    //Uncapable(crate::blockchain::Capability),
+    //MissingCachedAddresses,
     #[cfg(feature = "electrum")]
+    /// Electrum client error
     Electrum(electrum_client::Error),
     #[cfg(feature = "esplora")]
+    /// Esplora client error
     Esplora(crate::blockchain::esplora::EsploraError),
     #[cfg(feature = "compact_filters")]
+    /// Compact filters client error)
     CompactFilters(crate::blockchain::compact_filters::CompactFiltersError),
     #[cfg(feature = "key-value-db")]
+    /// Sled database error
     Sled(sled::Error),
+    #[cfg(feature = "rpc")]
+    /// Rpc client error
+    Rpc(bitcoincore_rpc::Error),
 }
 
 impl fmt::Display for Error {
@@ -95,32 +156,41 @@ impl std::error::Error for Error {}
 
 macro_rules! impl_error {
     ( $from:ty, $to:ident ) => {
-        impl std::convert::From<$from> for Error {
+        impl_error!($from, $to, Error);
+    };
+    ( $from:ty, $to:ident, $impl_for:ty ) => {
+        impl std::convert::From<$from> for $impl_for {
             fn from(err: $from) -> Self {
-                Error::$to(err)
+                <$impl_for>::$to(err)
             }
         }
     };
 }
 
-impl_error!(crate::descriptor::error::Error, Descriptor);
-impl_error!(
-    crate::wallet::address_validator::AddressValidatorError,
-    AddressValidator
-);
-impl_error!(
-    crate::descriptor::policy::PolicyError,
-    InvalidPolicyPathError
-);
-impl_error!(crate::wallet::signer::SignerError, Signer);
+impl_error!(descriptor::error::Error, Descriptor);
+impl_error!(address_validator::AddressValidatorError, AddressValidator);
+impl_error!(descriptor::policy::PolicyError, InvalidPolicyPathError);
+impl_error!(wallet::signer::SignerError, Signer);
+
+impl From<crate::keys::KeyError> for Error {
+    fn from(key_error: crate::keys::KeyError) -> Error {
+        match key_error {
+            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
+            crate::keys::KeyError::Bip32(inner) => Error::Bip32(inner),
+            crate::keys::KeyError::InvalidChecksum => Error::ChecksumMismatch,
+            e => Error::Key(e),
+        }
+    }
+}
 
 impl_error!(bitcoin::consensus::encode::Error, Encode);
 impl_error!(miniscript::Error, Miniscript);
-impl_error!(bitcoin::util::bip32::Error, BIP32);
+impl_error!(bitcoin::util::bip32::Error, Bip32);
 impl_error!(bitcoin::secp256k1::Error, Secp256k1);
-impl_error!(serde_json::Error, JSON);
+impl_error!(serde_json::Error, Json);
 impl_error!(bitcoin::hashes::hex::Error, Hex);
-impl_error!(bitcoin::util::psbt::Error, PSBT);
+impl_error!(bitcoin::util::psbt::Error, Psbt);
+impl_error!(bitcoin::util::psbt::PsbtParseError, PsbtParse);
 
 #[cfg(feature = "electrum")]
 impl_error!(electrum_client::Error, Electrum);
@@ -128,13 +198,25 @@ impl_error!(electrum_client::Error, Electrum);
 impl_error!(crate::blockchain::esplora::EsploraError, Esplora);
 #[cfg(feature = "key-value-db")]
 impl_error!(sled::Error, Sled);
+#[cfg(feature = "rpc")]
+impl_error!(bitcoincore_rpc::Error, Rpc);
 
 #[cfg(feature = "compact_filters")]
 impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
     fn from(other: crate::blockchain::compact_filters::CompactFiltersError) -> Self {
         match other {
             crate::blockchain::compact_filters::CompactFiltersError::Global(e) => *e,
-            err @ _ => Error::CompactFilters(err),
+            err => Error::CompactFilters(err),
+        }
+    }
+}
+
+#[cfg(feature = "verify")]
+impl From<crate::wallet::verify::VerifyError> for Error {
+    fn from(other: crate::wallet::verify::VerifyError) -> Self {
+        match other {
+            crate::wallet::verify::VerifyError::Global(inner) => *inner,
+            err => Error::Verification(err),
         }
     }
 }

src/keys/bip39.rs

@@ -0,0 +1,190 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! BIP-0039
+
+// TODO: maybe write our own implementation of bip39? Seems stupid to have an extra dependency for
+// something that should be fairly simple to re-implement.
+
+use bitcoin::util::bip32;
+use bitcoin::Network;
+
+use miniscript::ScriptContext;
+
+pub use bip39::{Language, Mnemonic, MnemonicType, Seed};
+
+use super::{
+    any_network, DerivableKey, DescriptorKey, ExtendedKey, GeneratableKey, GeneratedKey, KeyError,
+};
+
+fn set_valid_on_any_network<Ctx: ScriptContext>(
+    descriptor_key: DescriptorKey<Ctx>,
+) -> DescriptorKey<Ctx> {
+    // We have to pick one network to build the xprv, but since the bip39 standard doesn't
+    // encode the network, the xprv we create is actually valid everywhere. So we override the
+    // valid networks with `any_network()`.
+    descriptor_key.override_valid_networks(any_network())
+}
+
+/// Type for a BIP39 mnemonic with an optional passphrase
+pub type MnemonicWithPassphrase = (Mnemonic, Option<String>);
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for Seed {
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+        Ok(bip32::ExtendedPrivKey::new_master(Network::Bitcoin, &self.as_bytes())?.into())
+    }
+
+    fn into_descriptor_key(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let descriptor_key = self
+            .into_extended_key()?
+            .into_descriptor_key(source, derivation_path)?;
+
+        Ok(set_valid_on_any_network(descriptor_key))
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for MnemonicWithPassphrase {
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+        let (mnemonic, passphrase) = self;
+        let seed = Seed::new(&mnemonic, passphrase.as_deref().unwrap_or(""));
+
+        seed.into_extended_key()
+    }
+
+    fn into_descriptor_key(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let descriptor_key = self
+            .into_extended_key()?
+            .into_descriptor_key(source, derivation_path)?;
+
+        Ok(set_valid_on_any_network(descriptor_key))
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for Mnemonic {
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+        (self, None).into_extended_key()
+    }
+
+    fn into_descriptor_key(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let descriptor_key = self
+            .into_extended_key()?
+            .into_descriptor_key(source, derivation_path)?;
+
+        Ok(set_valid_on_any_network(descriptor_key))
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for Mnemonic {
+    type Entropy = [u8; 32];
+
+    type Options = (MnemonicType, Language);
+    type Error = Option<bip39::ErrorKind>;
+
+    fn generate_with_entropy(
+        (mnemonic_type, language): Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        let entropy = &entropy.as_ref()[..(mnemonic_type.entropy_bits() / 8)];
+        let mnemonic = Mnemonic::from_entropy(entropy, language).map_err(|e| e.downcast().ok())?;
+
+        Ok(GeneratedKey::new(mnemonic, any_network()))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+
+    use bitcoin::util::bip32;
+
+    use bip39::{Language, Mnemonic, MnemonicType};
+
+    use crate::keys::{any_network, GeneratableKey, GeneratedKey};
+
+    #[test]
+    fn test_keys_bip39_mnemonic() {
+        let mnemonic =
+            "aim bunker wash balance finish force paper analyst cabin spoon stable organ";
+        let mnemonic = Mnemonic::from_phrase(mnemonic, Language::English).unwrap();
+        let path = bip32::DerivationPath::from_str("m/44'/0'/0'/0").unwrap();
+
+        let key = (mnemonic, path);
+        let (desc, keys, networks) = crate::descriptor!(wpkh(key)).unwrap();
+        assert_eq!(desc.to_string(), "wpkh([be83839f/44'/0'/0']xpub6DCQ1YcqvZtSwGWMrwHELPehjWV3f2MGZ69yBADTxFEUAoLwb5Mp5GniQK6tTp3AgbngVz9zEFbBJUPVnkG7LFYt8QMTfbrNqs6FNEwAPKA/0/*)#0r8v4nkv");
+        assert_eq!(keys.len(), 1);
+        assert_eq!(networks.len(), 4);
+    }
+
+    #[test]
+    fn test_keys_bip39_mnemonic_passphrase() {
+        let mnemonic =
+            "aim bunker wash balance finish force paper analyst cabin spoon stable organ";
+        let mnemonic = Mnemonic::from_phrase(mnemonic, Language::English).unwrap();
+        let path = bip32::DerivationPath::from_str("m/44'/0'/0'/0").unwrap();
+
+        let key = ((mnemonic, Some("passphrase".into())), path);
+        let (desc, keys, networks) = crate::descriptor!(wpkh(key)).unwrap();
+        assert_eq!(desc.to_string(), "wpkh([8f6cb80c/44'/0'/0']xpub6DWYS8bbihFevy29M4cbw4ZR3P5E12jB8R88gBDWCTCNpYiDHhYWNywrCF9VZQYagzPmsZpxXpytzSoxynyeFr4ZyzheVjnpLKuse4fiwZw/0/*)#h0j0tg5m");
+        assert_eq!(keys.len(), 1);
+        assert_eq!(networks.len(), 4);
+    }
+
+    #[test]
+    fn test_keys_generate_bip39() {
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate_with_entropy(
+                (MnemonicType::Words12, Language::English),
+                crate::keys::test::TEST_ENTROPY,
+            )
+            .unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+        assert_eq!(
+            generated_mnemonic.to_string(),
+            "primary fetch primary fetch primary fetch primary fetch primary fetch primary fever"
+        );
+
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate_with_entropy(
+                (MnemonicType::Words24, Language::English),
+                crate::keys::test::TEST_ENTROPY,
+            )
+            .unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+        assert_eq!(generated_mnemonic.to_string(), "primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary foster");
+    }
+
+    #[test]
+    fn test_keys_generate_bip39_random() {
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate((MnemonicType::Words12, Language::English)).unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate((MnemonicType::Words24, Language::English)).unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+    }
+}

src/keys/mod.rs

@@ -0,0 +1,920 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Key formats
+
+use std::any::TypeId;
+use std::collections::HashSet;
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::str::FromStr;
+
+use bitcoin::secp256k1::{self, Secp256k1, Signing};
+
+use bitcoin::util::bip32;
+use bitcoin::{Network, PrivateKey, PublicKey};
+
+use miniscript::descriptor::{Descriptor, DescriptorXKey, Wildcard};
+pub use miniscript::descriptor::{
+    DescriptorPublicKey, DescriptorSecretKey, DescriptorSinglePriv, DescriptorSinglePub, KeyMap,
+    SortedMultiVec,
+};
+pub use miniscript::ScriptContext;
+use miniscript::{Miniscript, Terminal};
+
+use crate::descriptor::{CheckMiniscript, DescriptorError};
+use crate::wallet::utils::SecpCtx;
+
+#[cfg(feature = "keys-bip39")]
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+pub mod bip39;
+
+/// Set of valid networks for a key
+pub type ValidNetworks = HashSet<Network>;
+
+/// Create a set containing mainnet, testnet and regtest
+pub fn any_network() -> ValidNetworks {
+    vec![
+        Network::Bitcoin,
+        Network::Testnet,
+        Network::Regtest,
+        Network::Signet,
+    ]
+    .into_iter()
+    .collect()
+}
+/// Create a set only containing mainnet
+pub fn mainnet_network() -> ValidNetworks {
+    vec![Network::Bitcoin].into_iter().collect()
+}
+/// Create a set containing testnet and regtest
+pub fn test_networks() -> ValidNetworks {
+    vec![Network::Testnet, Network::Regtest, Network::Signet]
+        .into_iter()
+        .collect()
+}
+/// Compute the intersection of two sets
+pub fn merge_networks(a: &ValidNetworks, b: &ValidNetworks) -> ValidNetworks {
+    a.intersection(b).cloned().collect()
+}
+
+/// Container for public or secret keys
+#[derive(Debug)]
+pub enum DescriptorKey<Ctx: ScriptContext> {
+    #[doc(hidden)]
+    Public(DescriptorPublicKey, ValidNetworks, PhantomData<Ctx>),
+    #[doc(hidden)]
+    Secret(DescriptorSecretKey, ValidNetworks, PhantomData<Ctx>),
+}
+
+impl<Ctx: ScriptContext> DescriptorKey<Ctx> {
+    /// Create an instance given a public key and a set of valid networks
+    pub fn from_public(public: DescriptorPublicKey, networks: ValidNetworks) -> Self {
+        DescriptorKey::Public(public, networks, PhantomData)
+    }
+
+    /// Create an instance given a secret key and a set of valid networks
+    pub fn from_secret(secret: DescriptorSecretKey, networks: ValidNetworks) -> Self {
+        DescriptorKey::Secret(secret, networks, PhantomData)
+    }
+
+    /// Override the computed set of valid networks
+    pub fn override_valid_networks(self, networks: ValidNetworks) -> Self {
+        match self {
+            DescriptorKey::Public(key, _, _) => DescriptorKey::Public(key, networks, PhantomData),
+            DescriptorKey::Secret(key, _, _) => DescriptorKey::Secret(key, networks, PhantomData),
+        }
+    }
+
+    // This method is used internally by `bdk::fragment!` and `bdk::descriptor!`. It has to be
+    // public because it is effectively called by external crates, once the macros are expanded,
+    // but since it is not meant to be part of the public api we hide it from the docs.
+    #[doc(hidden)]
+    pub fn extract(
+        self,
+        secp: &SecpCtx,
+    ) -> Result<(DescriptorPublicKey, KeyMap, ValidNetworks), KeyError> {
+        match self {
+            DescriptorKey::Public(public, valid_networks, _) => {
+                Ok((public, KeyMap::default(), valid_networks))
+            }
+            DescriptorKey::Secret(secret, valid_networks, _) => {
+                let mut key_map = KeyMap::with_capacity(1);
+
+                let public = secret
+                    .as_public(secp)
+                    .map_err(|e| miniscript::Error::Unexpected(e.to_string()))?;
+                key_map.insert(public.clone(), secret);
+
+                Ok((public, key_map, valid_networks))
+            }
+        }
+    }
+}
+
+/// Enum representation of the known valid [`ScriptContext`]s
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub enum ScriptContextEnum {
+    /// Legacy scripts
+    Legacy,
+    /// Segwitv0 scripts
+    Segwitv0,
+}
+
+impl ScriptContextEnum {
+    /// Returns whether the script context is [`ScriptContextEnum::Legacy`]
+    pub fn is_legacy(&self) -> bool {
+        self == &ScriptContextEnum::Legacy
+    }
+
+    /// Returns whether the script context is [`ScriptContextEnum::Segwitv0`]
+    pub fn is_segwit_v0(&self) -> bool {
+        self == &ScriptContextEnum::Segwitv0
+    }
+}
+
+/// Trait that adds extra useful methods to [`ScriptContext`]s
+pub trait ExtScriptContext: ScriptContext {
+    /// Returns the [`ScriptContext`] as a [`ScriptContextEnum`]
+    fn as_enum() -> ScriptContextEnum;
+
+    /// Returns whether the script context is [`Legacy`](miniscript::Legacy)
+    fn is_legacy() -> bool {
+        Self::as_enum().is_legacy()
+    }
+
+    /// Returns whether the script context is [`Segwitv0`](miniscript::Segwitv0)
+    fn is_segwit_v0() -> bool {
+        Self::as_enum().is_segwit_v0()
+    }
+}
+
+impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
+    fn as_enum() -> ScriptContextEnum {
+        match TypeId::of::<Ctx>() {
+            t if t == TypeId::of::<miniscript::Legacy>() => ScriptContextEnum::Legacy,
+            t if t == TypeId::of::<miniscript::Segwitv0>() => ScriptContextEnum::Segwitv0,
+            _ => unimplemented!("Unknown ScriptContext type"),
+        }
+    }
+}
+
+/// Trait for objects that can be turned into a public or secret [`DescriptorKey`]
+///
+/// The generic type `Ctx` is used to define the context in which the key is valid: some key
+/// formats, like the mnemonics used by Electrum wallets, encode internally whether the wallet is
+/// legacy or segwit. Thus, trying to turn a valid legacy mnemonic into a `DescriptorKey`
+/// that would become part of a segwit descriptor should fail.
+///
+/// For key types that do care about this, the [`ExtScriptContext`] trait provides some useful
+/// methods that can be used to check at runtime which `Ctx` is being used.
+///
+/// For key types that can do this check statically (because they can only work within a
+/// single `Ctx`), the "specialized" trait can be implemented to make the compiler handle the type
+/// checking.
+///
+/// Keys also have control over the networks they support: constructing the return object with
+/// [`DescriptorKey::from_public`] or [`DescriptorKey::from_secret`] allows to specify a set of
+/// [`ValidNetworks`].
+///
+/// ## Examples
+///
+/// Key type valid in any context:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError, ScriptContext};
+///
+/// pub struct MyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for MyKeyType {
+///     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         self.pubkey.into_descriptor_key()
+///     }
+/// }
+/// ```
+///
+/// Key type that is only valid on mainnet:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{
+///     mainnet_network, DescriptorKey, DescriptorPublicKey, DescriptorSinglePub,
+///     IntoDescriptorKey, KeyError, ScriptContext,
+/// };
+///
+/// pub struct MyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for MyKeyType {
+///     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         Ok(DescriptorKey::from_public(
+///             DescriptorPublicKey::SinglePub(DescriptorSinglePub {
+///                 origin: None,
+///                 key: self.pubkey,
+///             }),
+///             mainnet_network(),
+///         ))
+///     }
+/// }
+/// ```
+///
+/// Key type that internally encodes in which context it's valid. The context is checked at runtime:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{DescriptorKey, ExtScriptContext, IntoDescriptorKey, KeyError, ScriptContext};
+///
+/// pub struct MyKeyType {
+///     is_legacy: bool,
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext + 'static> IntoDescriptorKey<Ctx> for MyKeyType {
+///     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         if Ctx::is_legacy() == self.is_legacy {
+///             self.pubkey.into_descriptor_key()
+///         } else {
+///             Err(KeyError::InvalidScriptContext)
+///         }
+///     }
+/// }
+/// ```
+///
+/// Key type that can only work within [`miniscript::Segwitv0`] context. Only the specialized version
+/// of the trait is implemented.
+///
+/// This example deliberately fails to compile, to demonstrate how the compiler can catch when keys
+/// are misused. In this case, the "segwit-only" key is used to build a `pkh()` descriptor, which
+/// makes the compiler (correctly) fail.
+///
+/// ```compile_fail
+/// use bdk::bitcoin::PublicKey;
+/// use std::str::FromStr;
+///
+/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError};
+///
+/// pub struct MySegwitOnlyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl IntoDescriptorKey<bdk::miniscript::Segwitv0> for MySegwitOnlyKeyType {
+///     fn into_descriptor_key(self) -> Result<DescriptorKey<bdk::miniscript::Segwitv0>, KeyError> {
+///         self.pubkey.into_descriptor_key()
+///     }
+/// }
+///
+/// let key = MySegwitOnlyKeyType {
+///     pubkey: PublicKey::from_str("...")?,
+/// };
+/// let (descriptor, _, _) = bdk::descriptor!(pkh(key))?;
+/// //                                       ^^^^^ changing this to `wpkh` would make it compile
+///
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub trait IntoDescriptorKey<Ctx: ScriptContext>: Sized {
+    /// Turn the key into a [`DescriptorKey`] within the requested [`ScriptContext`]
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError>;
+}
+
+/// Enum for extended keys that can be either `xprv` or `xpub`
+///
+/// An instance of [`ExtendedKey`] can be constructed from an [`ExtendedPrivKey`](bip32::ExtendedPrivKey)
+/// or an [`ExtendedPubKey`](bip32::ExtendedPubKey) by using the `From` trait.
+///
+/// Defaults to the [`Legacy`](miniscript::Legacy) context.
+pub enum ExtendedKey<Ctx: ScriptContext = miniscript::Legacy> {
+    /// A private extended key, aka an `xprv`
+    Private((bip32::ExtendedPrivKey, PhantomData<Ctx>)),
+    /// A public extended key, aka an `xpub`
+    Public((bip32::ExtendedPubKey, PhantomData<Ctx>)),
+}
+
+impl<Ctx: ScriptContext> ExtendedKey<Ctx> {
+    /// Return whether or not the key contains the private data
+    pub fn has_secret(&self) -> bool {
+        match self {
+            ExtendedKey::Private(_) => true,
+            ExtendedKey::Public(_) => false,
+        }
+    }
+
+    /// Transform the [`ExtendedKey`] into an [`ExtendedPrivKey`](bip32::ExtendedPrivKey) for the
+    /// given [`Network`], if the key contains the private data
+    pub fn into_xprv(self, network: Network) -> Option<bip32::ExtendedPrivKey> {
+        match self {
+            ExtendedKey::Private((mut xprv, _)) => {
+                xprv.network = network;
+                Some(xprv)
+            }
+            ExtendedKey::Public(_) => None,
+        }
+    }
+
+    /// Transform the [`ExtendedKey`] into an [`ExtendedPubKey`](bip32::ExtendedPubKey) for the
+    /// given [`Network`]
+    pub fn into_xpub<C: Signing>(
+        self,
+        network: bitcoin::Network,
+        secp: &Secp256k1<C>,
+    ) -> bip32::ExtendedPubKey {
+        let mut xpub = match self {
+            ExtendedKey::Private((xprv, _)) => bip32::ExtendedPubKey::from_private(secp, &xprv),
+            ExtendedKey::Public((xpub, _)) => xpub,
+        };
+
+        xpub.network = network;
+        xpub
+    }
+}
+
+impl<Ctx: ScriptContext> From<bip32::ExtendedPubKey> for ExtendedKey<Ctx> {
+    fn from(xpub: bip32::ExtendedPubKey) -> Self {
+        ExtendedKey::Public((xpub, PhantomData))
+    }
+}
+
+impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
+    fn from(xprv: bip32::ExtendedPrivKey) -> Self {
+        ExtendedKey::Private((xprv, PhantomData))
+    }
+}
+
+/// Trait for keys that can be derived.
+///
+/// When extra metadata are provided, a [`DerivableKey`] can be transofrmed into a
+/// [`DescriptorKey`]: the trait [`IntoDescriptorKey`] is automatically implemented
+/// for `(DerivableKey, DerivationPath)` and
+/// `(DerivableKey, KeySource, DerivationPath)` tuples.
+///
+/// For key types that don't encode any indication about the path to use (like bip39), it's
+/// generally recommended to implemented this trait instead of [`IntoDescriptorKey`]. The same
+/// rules regarding script context and valid networks apply.
+///
+/// ## Examples
+///
+/// Key types that can be directly converted into an [`ExtendedPrivKey`] or
+/// an [`ExtendedPubKey`] can implement only the required `into_extended_key()` method.
+///
+/// ```
+/// use bdk::bitcoin;
+/// use bdk::bitcoin::util::bip32;
+/// use bdk::keys::{DerivableKey, ExtendedKey, KeyError, ScriptContext};
+///
+/// struct MyCustomKeyType {
+///     key_data: bitcoin::PrivateKey,
+///     chain_code: Vec<u8>,
+///     network: bitcoin::Network,
+/// }
+///
+/// impl<Ctx: ScriptContext> DerivableKey<Ctx> for MyCustomKeyType {
+///     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+///         let xprv = bip32::ExtendedPrivKey {
+///             network: self.network,
+///             depth: 0,
+///             parent_fingerprint: bip32::Fingerprint::default(),
+///             private_key: self.key_data,
+///             chain_code: bip32::ChainCode::from(self.chain_code.as_ref()),
+///             child_number: bip32::ChildNumber::Normal { index: 0 },
+///         };
+///
+///         xprv.into_extended_key()
+///     }
+/// }
+/// ```
+///
+/// Types that don't internally encode the [`Network`](bitcoin::Network) in which they are valid need some extra
+/// steps to override the set of valid networks, otherwise only the network specified in the
+/// [`ExtendedPrivKey`] or [`ExtendedPubKey`] will be considered valid.
+///
+/// ```
+/// use bdk::bitcoin;
+/// use bdk::bitcoin::util::bip32;
+/// use bdk::keys::{
+///     any_network, DerivableKey, DescriptorKey, ExtendedKey, KeyError, ScriptContext,
+/// };
+///
+/// struct MyCustomKeyType {
+///     key_data: bitcoin::PrivateKey,
+///     chain_code: Vec<u8>,
+/// }
+///
+/// impl<Ctx: ScriptContext> DerivableKey<Ctx> for MyCustomKeyType {
+///     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+///         let xprv = bip32::ExtendedPrivKey {
+///             network: bitcoin::Network::Bitcoin, // pick an arbitrary network here
+///             depth: 0,
+///             parent_fingerprint: bip32::Fingerprint::default(),
+///             private_key: self.key_data,
+///             chain_code: bip32::ChainCode::from(self.chain_code.as_ref()),
+///             child_number: bip32::ChildNumber::Normal { index: 0 },
+///         };
+///
+///         xprv.into_extended_key()
+///     }
+///
+///     fn into_descriptor_key(
+///         self,
+///         source: Option<bip32::KeySource>,
+///         derivation_path: bip32::DerivationPath,
+///     ) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         let descriptor_key = self
+///             .into_extended_key()?
+///             .into_descriptor_key(source, derivation_path)?;
+///
+///         // Override the set of valid networks here
+///         Ok(descriptor_key.override_valid_networks(any_network()))
+///     }
+/// }
+/// ```
+///
+/// [`DerivationPath`]: (bip32::DerivationPath)
+/// [`ExtendedPrivKey`]: (bip32::ExtendedPrivKey)
+/// [`ExtendedPubKey`]: (bip32::ExtendedPubKey)
+pub trait DerivableKey<Ctx: ScriptContext = miniscript::Legacy>: Sized {
+    /// Consume `self` and turn it into an [`ExtendedKey`]
+    ///
+    /// This can be used to get direct access to `xprv`s and `xpub`s for types that implement this trait,
+    /// like [`Mnemonic`](bip39::Mnemonic) when the `keys-bip39` feature is enabled.
+    #[cfg_attr(
+        feature = "keys-bip39",
+        doc = r##"
+```rust
+use bdk::bitcoin::Network;
+use bdk::keys::{DerivableKey, ExtendedKey};
+use bdk::keys::bip39::{Mnemonic, Language};
+
+# fn main() -> Result<(), Box<dyn std::error::Error>> {
+let xkey: ExtendedKey =
+    Mnemonic::from_phrase(
+        "jelly crash boy whisper mouse ecology tuna soccer memory million news short",
+        Language::English
+    )?
+    .into_extended_key()?;
+let xprv = xkey.into_xprv(Network::Bitcoin).unwrap();
+# Ok(()) }
+```
+"##
+    )]
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError>;
+
+    /// Consume `self` and turn it into a [`DescriptorKey`] by adding the extra metadata, such as
+    /// key origin and derivation path
+    fn into_descriptor_key(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        match self.into_extended_key()? {
+            ExtendedKey::Private((xprv, _)) => DescriptorSecretKey::XPrv(DescriptorXKey {
+                origin,
+                xkey: xprv,
+                derivation_path,
+                wildcard: Wildcard::Unhardened,
+            })
+            .into_descriptor_key(),
+            ExtendedKey::Public((xpub, _)) => DescriptorPublicKey::XPub(DescriptorXKey {
+                origin,
+                xkey: xpub,
+                derivation_path,
+                wildcard: Wildcard::Unhardened,
+            })
+            .into_descriptor_key(),
+        }
+    }
+}
+
+/// Identity conversion
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for ExtendedKey<Ctx> {
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+        Ok(self)
+    }
+}
+
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPubKey {
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+        Ok(self.into())
+    }
+}
+
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPrivKey {
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+        Ok(self.into())
+    }
+}
+
+/// Output of a [`GeneratableKey`] key generation
+pub struct GeneratedKey<K, Ctx: ScriptContext> {
+    key: K,
+    valid_networks: ValidNetworks,
+    phantom: PhantomData<Ctx>,
+}
+
+impl<K, Ctx: ScriptContext> GeneratedKey<K, Ctx> {
+    fn new(key: K, valid_networks: ValidNetworks) -> Self {
+        GeneratedKey {
+            key,
+            valid_networks,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Consumes `self` and returns the key
+    pub fn into_key(self) -> K {
+        self.key
+    }
+}
+
+impl<K, Ctx: ScriptContext> Deref for GeneratedKey<K, Ctx> {
+    type Target = K;
+
+    fn deref(&self) -> &Self::Target {
+        &self.key
+    }
+}
+
+// Make generated "derivable" keys themselves "derivable". Also make sure they are assigned the
+// right `valid_networks`.
+impl<Ctx, K> DerivableKey<Ctx> for GeneratedKey<K, Ctx>
+where
+    Ctx: ScriptContext,
+    K: DerivableKey<Ctx>,
+{
+    fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
+        self.key.into_extended_key()
+    }
+
+    fn into_descriptor_key(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let descriptor_key = self.key.into_descriptor_key(origin, derivation_path)?;
+        Ok(descriptor_key.override_valid_networks(self.valid_networks))
+    }
+}
+
+// Make generated keys directly usable in descriptors, and make sure they get assigned the right
+// `valid_networks`.
+impl<Ctx, K> IntoDescriptorKey<Ctx> for GeneratedKey<K, Ctx>
+where
+    Ctx: ScriptContext,
+    K: IntoDescriptorKey<Ctx>,
+{
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let desc_key = self.key.into_descriptor_key()?;
+        Ok(desc_key.override_valid_networks(self.valid_networks))
+    }
+}
+
+/// Trait for keys that can be generated
+///
+/// The same rules about [`ScriptContext`] and [`ValidNetworks`] from [`IntoDescriptorKey`] apply.
+///
+/// This trait is particularly useful when combined with [`DerivableKey`]: if `Self`
+/// implements it, the returned [`GeneratedKey`] will also implement it. The same is true for
+/// [`IntoDescriptorKey`]: the generated keys can be directly used in descriptors if `Self` is also
+/// [`IntoDescriptorKey`].
+pub trait GeneratableKey<Ctx: ScriptContext>: Sized {
+    /// Type specifying the amount of entropy required e.g. `[u8;32]`
+    type Entropy: AsMut<[u8]> + Default;
+
+    /// Extra options required by the `generate_with_entropy`
+    type Options;
+    /// Returned error in case of failure
+    type Error: std::fmt::Debug;
+
+    /// Generate a key given the extra options and the entropy
+    fn generate_with_entropy(
+        options: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error>;
+
+    /// Generate a key given the options with a random entropy
+    fn generate(options: Self::Options) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        use rand::{thread_rng, Rng};
+
+        let mut entropy = Self::Entropy::default();
+        thread_rng().fill(entropy.as_mut());
+        Self::generate_with_entropy(options, entropy)
+    }
+}
+
+/// Trait that allows generating a key with the default options
+///
+/// This trait is automatically implemented if the [`GeneratableKey::Options`] implements [`Default`].
+pub trait GeneratableDefaultOptions<Ctx>: GeneratableKey<Ctx>
+where
+    Ctx: ScriptContext,
+    <Self as GeneratableKey<Ctx>>::Options: Default,
+{
+    /// Generate a key with the default options and a given entropy
+    fn generate_with_entropy_default(
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        Self::generate_with_entropy(Default::default(), entropy)
+    }
+
+    /// Generate a key with the default options and a random entropy
+    fn generate_default() -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        Self::generate(Default::default())
+    }
+}
+
+/// Automatic implementation of [`GeneratableDefaultOptions`] for [`GeneratableKey`]s where
+/// `Options` implements `Default`
+impl<Ctx, K> GeneratableDefaultOptions<Ctx> for K
+where
+    Ctx: ScriptContext,
+    K: GeneratableKey<Ctx>,
+    <K as GeneratableKey<Ctx>>::Options: Default,
+{
+}
+
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::ExtendedPrivKey {
+    type Entropy = [u8; 32];
+
+    type Options = ();
+    type Error = bip32::Error;
+
+    fn generate_with_entropy(
+        _: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        // pick a arbitrary network here, but say that we support all of them
+        let xprv = bip32::ExtendedPrivKey::new_master(Network::Bitcoin, entropy.as_ref())?;
+        Ok(GeneratedKey::new(xprv, any_network()))
+    }
+}
+
+/// Options for generating a [`PrivateKey`]
+///
+/// Defaults to creating compressed keys, which save on-chain bytes and fees
+#[derive(Debug, Copy, Clone)]
+pub struct PrivateKeyGenerateOptions {
+    /// Whether the generated key should be "compressed" or not
+    pub compressed: bool,
+}
+
+impl Default for PrivateKeyGenerateOptions {
+    fn default() -> Self {
+        PrivateKeyGenerateOptions { compressed: true }
+    }
+}
+
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for PrivateKey {
+    type Entropy = [u8; secp256k1::constants::SECRET_KEY_SIZE];
+
+    type Options = PrivateKeyGenerateOptions;
+    type Error = bip32::Error;
+
+    fn generate_with_entropy(
+        options: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        // pick a arbitrary network here, but say that we support all of them
+        let key = secp256k1::SecretKey::from_slice(&entropy)?;
+        let private_key = PrivateKey {
+            compressed: options.compressed,
+            network: Network::Bitcoin,
+            key,
+        };
+
+        Ok(GeneratedKey::new(private_key, any_network()))
+    }
+}
+
+impl<Ctx: ScriptContext, T: DerivableKey<Ctx>> IntoDescriptorKey<Ctx>
+    for (T, bip32::DerivationPath)
+{
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        self.0.into_descriptor_key(None, self.1)
+    }
+}
+
+impl<Ctx: ScriptContext, T: DerivableKey<Ctx>> IntoDescriptorKey<Ctx>
+    for (T, bip32::KeySource, bip32::DerivationPath)
+{
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        self.0.into_descriptor_key(Some(self.1), self.2)
+    }
+}
+
+fn expand_multi_keys<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<(Vec<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError> {
+    let (pks, key_maps_networks): (Vec<_>, Vec<_>) = pks
+        .into_iter()
+        .map(|key| key.into_descriptor_key()?.extract(secp))
+        .collect::<Result<Vec<_>, _>>()?
+        .into_iter()
+        .map(|(a, b, c)| (a, (b, c)))
+        .unzip();
+
+    let (key_map, valid_networks) = key_maps_networks.into_iter().fold(
+        (KeyMap::default(), any_network()),
+        |(mut keys_acc, net_acc), (key, net)| {
+            keys_acc.extend(key.into_iter());
+            let net_acc = merge_networks(&net_acc, &net);
+
+            (keys_acc, net_acc)
+        },
+    );
+
+    Ok((pks, key_map, valid_networks))
+}
+
+// Used internally by `bdk::fragment!` to build `pk_k()` fragments
+#[doc(hidden)]
+pub fn make_pk<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    descriptor_key: Pk,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), DescriptorError> {
+    let (key, key_map, valid_networks) = descriptor_key.into_descriptor_key()?.extract(secp)?;
+    let minisc = Miniscript::from_ast(Terminal::PkK(key))?;
+
+    minisc.check_minsicript()?;
+
+    Ok((minisc, key_map, valid_networks))
+}
+
+// Used internally by `bdk::fragment!` to build `multi()` fragments
+#[doc(hidden)]
+pub fn make_multi<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    thresh: usize,
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), DescriptorError> {
+    let (pks, key_map, valid_networks) = expand_multi_keys(pks, secp)?;
+    let minisc = Miniscript::from_ast(Terminal::Multi(thresh, pks))?;
+
+    minisc.check_minsicript()?;
+
+    Ok((minisc, key_map, valid_networks))
+}
+
+// Used internally by `bdk::descriptor!` to build `sortedmulti()` fragments
+#[doc(hidden)]
+pub fn make_sortedmulti<Pk, Ctx, F>(
+    thresh: usize,
+    pks: Vec<Pk>,
+    build_desc: F,
+    secp: &SecpCtx,
+) -> Result<(Descriptor<DescriptorPublicKey>, KeyMap, ValidNetworks), DescriptorError>
+where
+    Pk: IntoDescriptorKey<Ctx>,
+    Ctx: ScriptContext,
+    F: Fn(
+        usize,
+        Vec<DescriptorPublicKey>,
+    ) -> Result<(Descriptor<DescriptorPublicKey>, PhantomData<Ctx>), DescriptorError>,
+{
+    let (pks, key_map, valid_networks) = expand_multi_keys(pks, secp)?;
+    let descriptor = build_desc(thresh, pks)?.0;
+
+    Ok((descriptor, key_map, valid_networks))
+}
+
+/// The "identity" conversion is used internally by some `bdk::fragment`s
+impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorKey<Ctx> {
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        Ok(self)
+    }
+}
+
+impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorPublicKey {
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let networks = match self {
+            DescriptorPublicKey::SinglePub(_) => any_network(),
+            DescriptorPublicKey::XPub(DescriptorXKey { xkey, .. })
+                if xkey.network == Network::Bitcoin =>
+            {
+                mainnet_network()
+            }
+            _ => test_networks(),
+        };
+
+        Ok(DescriptorKey::from_public(self, networks))
+    }
+}
+
+impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for PublicKey {
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorPublicKey::SinglePub(DescriptorSinglePub {
+            key: self,
+            origin: None,
+        })
+        .into_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorSecretKey {
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let networks = match &self {
+            DescriptorSecretKey::SinglePriv(sk) if sk.key.network == Network::Bitcoin => {
+                mainnet_network()
+            }
+            DescriptorSecretKey::XPrv(DescriptorXKey { xkey, .. })
+                if xkey.network == Network::Bitcoin =>
+            {
+                mainnet_network()
+            }
+            _ => test_networks(),
+        };
+
+        Ok(DescriptorKey::from_secret(self, networks))
+    }
+}
+
+impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for &'_ str {
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::from_str(self)
+            .map_err(|e| KeyError::Message(e.to_string()))?
+            .into_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for PrivateKey {
+    fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::SinglePriv(DescriptorSinglePriv {
+            key: self,
+            origin: None,
+        })
+        .into_descriptor_key()
+    }
+}
+
+/// Errors thrown while working with [`keys`](crate::keys)
+#[derive(Debug)]
+pub enum KeyError {
+    /// The key cannot exist in the given script context
+    InvalidScriptContext,
+    /// The key is not valid for the given network
+    InvalidNetwork,
+    /// The key has an invalid checksum
+    InvalidChecksum,
+
+    /// Custom error message
+    Message(String),
+
+    /// BIP32 error
+    Bip32(bitcoin::util::bip32::Error),
+    /// Miniscript error
+    Miniscript(miniscript::Error),
+}
+
+impl_error!(miniscript::Error, Miniscript, KeyError);
+impl_error!(bitcoin::util::bip32::Error, Bip32, KeyError);
+
+impl std::fmt::Display for KeyError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for KeyError {}
+
+#[cfg(test)]
+pub mod test {
+    use bitcoin::util::bip32;
+
+    use super::*;
+
+    pub const TEST_ENTROPY: [u8; 32] = [0xAA; 32];
+
+    #[test]
+    fn test_keys_generate_xprv() {
+        let generated_xprv: GeneratedKey<_, miniscript::Segwitv0> =
+            bip32::ExtendedPrivKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+
+        assert_eq!(generated_xprv.valid_networks, any_network());
+        assert_eq!(generated_xprv.to_string(), "xprv9s21ZrQH143K4Xr1cJyqTvuL2FWR8eicgY9boWqMBv8MDVUZ65AXHnzBrK1nyomu6wdcabRgmGTaAKawvhAno1V5FowGpTLVx3jxzE5uk3Q");
+    }
+
+    #[test]
+    fn test_keys_generate_wif() {
+        let generated_wif: GeneratedKey<_, miniscript::Segwitv0> =
+            bitcoin::PrivateKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+
+        assert_eq!(generated_wif.valid_networks, any_network());
+        assert_eq!(
+            generated_wif.to_string(),
+            "L2wTu6hQrnDMiFNWA5na6jB12ErGQqtXwqpSL7aWquJaZG8Ai3ch"
+        );
+    }
+}

src/lib.rs

@@ -1,33 +1,202 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
+// rustdoc will warn if there are missing docs
+#![warn(missing_docs)]
 // only enables the `doc_cfg` feature when
 // the `docsrs` configuration attribute is defined
 #![cfg_attr(docsrs, feature(doc_cfg))]
-// only enables the nightly `external_doc` feature when
-// `test-md-docs` is enabled
-#![cfg_attr(feature = "test-md-docs", feature(external_doc))]
+
+//! A modern, lightweight, descriptor-based wallet library written in Rust.
+//!
+//! # About
+//!
+//! The BDK library aims to be the core building block for Bitcoin wallets of any kind.
+//!
+//! * It uses [Miniscript](https://github.com/rust-bitcoin/rust-miniscript) to support descriptors with generalized conditions. This exact same library can be used to build
+//!   single-sig wallets, multisigs, timelocked contracts and more.
+//! * It supports multiple blockchain backends and databases, allowing developers to choose exactly what's right for their projects.
+//! * It is built to be cross-platform: the core logic works on desktop, mobile, and even WebAssembly.
+//! * It is very easy to extend: developers can implement customized logic for blockchain backends, databases, signers, coin selection, and more, without having to fork and modify this library.
+//!
+//! # A Tour of BDK
+//!
+//! BDK consists of a number of modules that provide a range of functionality
+//! essential for implementing descriptor based Bitcoin wallet applications in Rust. In this
+//! section, we will take a brief tour of BDK, summarizing the major APIs and
+//! their uses.
+//!
+//! The easiest way to get started is to add bdk to your dependencies with the default features.
+//! The default features include a simple key-value database ([`sled`](sled)) to cache
+//! blockchain data and an [electrum](https://docs.rs/electrum-client/) blockchain client to
+//! interact with the bitcoin P2P network.
+//!
+//! ```toml
+//! bdk = "0.9.0"
+//! ```
+#![cfg_attr(
+    feature = "electrum",
+    doc = r##"
+## Sync the balance of a descriptor
+
+### Example
+```no_run
+use bdk::Wallet;
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+use bdk::electrum_client::Client;
+
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+    let wallet = Wallet::new(
+        "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+        ElectrumBlockchain::from(client)
+    )?;
+
+    wallet.sync(noop_progress(), None)?;
+
+    println!("Descriptor balance: {} SAT", wallet.get_balance()?);
+
+    Ok(())
+}
+```
+"##
+)]
+//!
+//! ## Generate a few addresses
+//!
+//! ### Example
+//! ```
+//! use bdk::{Wallet};
+//! use bdk::database::MemoryDatabase;
+//! use bdk::wallet::AddressIndex::New;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//! let wallet = Wallet::new_offline(
+//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!     )?;
+//!
+//!     println!("Address #0: {}", wallet.get_address(New)?);
+//!     println!("Address #1: {}", wallet.get_address(New)?);
+//!     println!("Address #2: {}", wallet.get_address(New)?);
+//!
+//!     Ok(())
+//! }
+//! ```
+#![cfg_attr(
+    feature = "electrum",
+    doc = r##"
+## Create a transaction
+
+### Example
+```no_run
+use bdk::{FeeRate, Wallet};
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+use bdk::electrum_client::Client;
+
+use bitcoin::consensus::serialize;
+use bdk::wallet::AddressIndex::New;
+
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+    let wallet = Wallet::new(
+        "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+        ElectrumBlockchain::from(client)
+    )?;
+
+    wallet.sync(noop_progress(), None)?;
+
+    let send_to = wallet.get_address(New)?;
+    let (psbt, details) = {
+        let mut builder =  wallet.build_tx();
+        builder
+            .add_recipient(send_to.script_pubkey(), 50_000)
+            .enable_rbf()
+            .do_not_spend_change()
+            .fee_rate(FeeRate::from_sat_per_vb(5.0));
+        builder.finish()?
+    };
+
+    println!("Transaction details: {:#?}", details);
+    println!("Unsigned PSBT: {}", &psbt);
+
+    Ok(())
+}
+```
+"##
+)]
+//!
+//! ## Sign a transaction
+//!
+//! ### Example
+//! ```no_run
+//! use std::str::FromStr;
+//!
+//! use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
+//!
+//! use bdk::{Wallet, SignOptions};
+//! use bdk::database::MemoryDatabase;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let wallet = Wallet::new_offline(
+//!         "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!     )?;
+//!
+//!     let psbt = "...";
+//!     let mut psbt = Psbt::from_str(psbt)?;
+//!
+//!     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # Feature flags
+//!
+//! BDK uses a set of [feature flags](https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section)
+//! to reduce the amount of compiled code by allowing projects to only enable the features they need.
+//! By default, BDK enables two internal features, `key-value-db` and `electrum`.
+//!
+//! If you are new to BDK we recommended that you use the default features which will enable
+//! basic descriptor wallet functionality. More advanced users can disable the `default` features
+//! (`--no-default-features`) and build the BDK library with only the features you need.
+
+//! Below is a list of the available feature flags and the additional functionality they provide.
+//!
+//! * `all-keys`: all features for working with bitcoin keys
+//! * `async-interface`: async functions in bdk traits
+//! * `keys-bip39`: [BIP-39](https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki) mnemonic codes for generating deterministic keys
+//!
+//! ## Internal features
+//!
+//! These features do not expose any new API, but influence internal implementation aspects of
+//! BDK.
+//!
+//! * `compact_filters`: [`compact_filters`](crate::blockchain::compact_filters) client protocol for interacting with the bitcoin P2P network
+//! * `electrum`: [`electrum`](crate::blockchain::electrum) client protocol for interacting with electrum servers
+//! * `esplora`: [`esplora`](crate::blockchain::esplora) client protocol for interacting with blockstream [electrs](https://github.com/Blockstream/electrs) servers
+//! * `key-value-db`: key value [`database`](crate::database) based on [`sled`](crate::sled) for caching blockchain data
 
 pub extern crate bitcoin;
 extern crate log;
@@ -36,16 +205,26 @@ extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
+#[cfg(all(feature = "async-interface", feature = "electrum"))]
+compile_error!(
+    "Features async-interface and electrum are mutually exclusive and cannot be enabled together"
+);
+
+#[cfg(feature = "keys-bip39")]
+extern crate bip39;
+
 #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
 #[macro_use]
 extern crate async_trait;
 #[macro_use]
-extern crate magical_macros;
+extern crate bdk_macros;
 
-#[cfg(any(test, feature = "compact_filters"))]
-#[macro_use]
+#[cfg(feature = "compact_filters")]
 extern crate lazy_static;
 
+#[cfg(feature = "rpc")]
+pub extern crate bitcoincore_rpc;
+
 #[cfg(feature = "electrum")]
 pub extern crate electrum_client;
 
@@ -55,18 +234,12 @@ pub extern crate reqwest;
 #[cfg(feature = "key-value-db")]
 pub extern crate sled;
 
-#[cfg(feature = "cli-utils")]
-pub mod cli;
-
+#[allow(unused_imports)]
+#[cfg(test)]
+#[allow(unused_imports)]
 #[cfg(test)]
 #[macro_use]
-extern crate testutils;
-#[cfg(test)]
-#[macro_use]
-extern crate testutils_macros;
-#[cfg(test)]
-#[macro_use]
-extern crate serial_test;
+pub extern crate serial_test;
 
 #[macro_use]
 pub(crate) mod error;
@@ -75,14 +248,29 @@ pub mod database;
 pub mod descriptor;
 #[cfg(feature = "test-md-docs")]
 mod doctest;
+pub mod keys;
 pub(crate) mod psbt;
 pub(crate) mod types;
 pub mod wallet;
 
-pub use descriptor::HDKeyPaths;
+pub use descriptor::template;
+pub use descriptor::HdKeyPaths;
 pub use error::Error;
 pub use types::*;
 pub use wallet::address_validator;
 pub use wallet::signer;
+pub use wallet::signer::SignOptions;
 pub use wallet::tx_builder::TxBuilder;
-pub use wallet::{OfflineWallet, Wallet};
+pub use wallet::Wallet;
+
+/// Get the version of BDK at runtime
+pub fn version() -> &'static str {
+    env!("CARGO_PKG_VERSION", "unknown")
+}
+
+// We should consider putting this under a feature flag but we need the macro in doctets so we need
+// to wait until https://github.com/rust-lang/rust/issues/67295 is fixed.
+//
+// Stuff in here is too rough to document atm
+#[doc(hidden)]
+pub mod testutils;

src/psbt/mod.rs

@@ -1,35 +1,23 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
-use bitcoin::util::psbt::PartiallySignedTransaction as PSBT;
+use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
 use bitcoin::TxOut;
 
-pub trait PSBTUtils {
+pub trait PsbtUtils {
     fn get_utxo_for(&self, input_index: usize) -> Option<TxOut>;
 }
 
-impl PSBTUtils for PSBT {
+impl PsbtUtils for Psbt {
+    #[allow(clippy::all)] // We want to allow `manual_map` but it is too new.
     fn get_utxo_for(&self, input_index: usize) -> Option<TxOut> {
         let tx = &self.global.unsigned_tx;
 
@@ -50,3 +38,85 @@ impl PSBTUtils for PSBT {
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    use crate::bitcoin::TxIn;
+    use crate::psbt::Psbt;
+    use crate::wallet::test::{get_funded_wallet, get_test_wpkh};
+    use crate::wallet::AddressIndex;
+    use crate::SignOptions;
+    use std::str::FromStr;
+
+    // from bip 174
+    const PSBT_STR: &str = "cHNidP8BAKACAAAAAqsJSaCMWvfEm4IS9Bfi8Vqz9cM9zxU4IagTn4d6W3vkAAAAAAD+////qwlJoIxa98SbghL0F+LxWrP1wz3PFTghqBOfh3pbe+QBAAAAAP7///8CYDvqCwAAAAAZdqkUdopAu9dAy+gdmI5x3ipNXHE5ax2IrI4kAAAAAAAAGXapFG9GILVT+glechue4O/p+gOcykWXiKwAAAAAAAEHakcwRAIgR1lmF5fAGwNrJZKJSGhiGDR9iYZLcZ4ff89X0eURZYcCIFMJ6r9Wqk2Ikf/REf3xM286KdqGbX+EhtdVRs7tr5MZASEDXNxh/HupccC1AaZGoqg7ECy0OIEhfKaC3Ibi1z+ogpIAAQEgAOH1BQAAAAAXqRQ1RebjO4MsRwUPJNPuuTycA5SLx4cBBBYAFIXRNTfy4mVAWjTbr6nj3aAfuCMIAAAA";
+
+    #[test]
+    #[should_panic(expected = "InputIndexOutOfRange")]
+    fn test_psbt_malformed_psbt_input_legacy() {
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+        psbt.inputs.push(psbt_bip.inputs[0].clone());
+        let options = SignOptions {
+            trust_witness_utxo: true,
+            ..Default::default()
+        };
+        let _ = wallet.sign(&mut psbt, options).unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "InputIndexOutOfRange")]
+    fn test_psbt_malformed_psbt_input_segwit() {
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+        psbt.inputs.push(psbt_bip.inputs[1].clone());
+        let options = SignOptions {
+            trust_witness_utxo: true,
+            ..Default::default()
+        };
+        let _ = wallet.sign(&mut psbt, options).unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "InputIndexOutOfRange")]
+    fn test_psbt_malformed_tx_input() {
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+        psbt.global.unsigned_tx.input.push(TxIn::default());
+        let options = SignOptions {
+            trust_witness_utxo: true,
+            ..Default::default()
+        };
+        let _ = wallet.sign(&mut psbt, options).unwrap();
+    }
+
+    #[test]
+    fn test_psbt_sign_with_finalized() {
+        let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
+        let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        let send_to = wallet.get_address(AddressIndex::New).unwrap();
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(send_to.script_pubkey(), 10_000);
+        let (mut psbt, _) = builder.finish().unwrap();
+
+        // add a finalized input
+        psbt.inputs.push(psbt_bip.inputs[0].clone());
+        psbt.global
+            .unsigned_tx
+            .input
+            .push(psbt_bip.global.unsigned_tx.input[0].clone());
+
+        let _ = wallet.sign(&mut psbt, SignOptions::default()).unwrap();
+    }
+}

src/testutils/blockchain_tests.rs

@@ -0,0 +1,879 @@
+use crate::testutils::TestIncomingTx;
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::sha256d;
+use bitcoin::{Address, Amount, Script, Transaction, Txid};
+pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
+pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
+use core::str::FromStr;
+pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
+#[allow(unused_imports)]
+use log::{debug, error, info, trace};
+use std::collections::HashMap;
+use std::env;
+use std::ops::Deref;
+use std::path::PathBuf;
+use std::time::Duration;
+
+pub struct TestClient {
+    client: RpcClient,
+    electrum: ElectrumClient,
+}
+
+impl TestClient {
+    pub fn new(rpc_host_and_wallet: String, rpc_wallet_name: String) -> Self {
+        let client = RpcClient::new(
+            format!("http://{}/wallet/{}", rpc_host_and_wallet, rpc_wallet_name),
+            get_auth(),
+        )
+        .unwrap();
+        let electrum = ElectrumClient::new(&get_electrum_url()).unwrap();
+
+        TestClient { client, electrum }
+    }
+
+    fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
+        // wait for electrs to index the tx
+        exponential_backoff_poll(|| {
+            trace!("wait_for_tx {}", txid);
+
+            self.electrum
+                .script_get_history(monitor_script)
+                .unwrap()
+                .iter()
+                .position(|entry| entry.tx_hash == txid)
+        });
+    }
+
+    fn wait_for_block(&mut self, min_height: usize) {
+        self.electrum.block_headers_subscribe().unwrap();
+
+        loop {
+            let header = exponential_backoff_poll(|| {
+                self.electrum.ping().unwrap();
+                self.electrum.block_headers_pop().unwrap()
+            });
+            if header.height >= min_height {
+                break;
+            }
+        }
+    }
+
+    pub fn receive(&mut self, meta_tx: TestIncomingTx) -> Txid {
+        assert!(
+            !meta_tx.output.is_empty(),
+            "can't create a transaction with no outputs"
+        );
+
+        let mut map = HashMap::new();
+
+        let mut required_balance = 0;
+        for out in &meta_tx.output {
+            required_balance += out.value;
+            map.insert(out.to_address.clone(), Amount::from_sat(out.value));
+        }
+
+        if self.get_balance(None, None).unwrap() < Amount::from_sat(required_balance) {
+            panic!("Insufficient funds in bitcoind. Please generate a few blocks with: `bitcoin-cli generatetoaddress 10 {}`", self.get_new_address(None, None).unwrap());
+        }
+
+        // FIXME: core can't create a tx with two outputs to the same address
+        let tx = self
+            .create_raw_transaction_hex(&[], &map, meta_tx.locktime, meta_tx.replaceable)
+            .unwrap();
+        let tx = self.fund_raw_transaction(tx, None, None).unwrap();
+        let mut tx: Transaction = deserialize(&tx.hex).unwrap();
+
+        if let Some(true) = meta_tx.replaceable {
+            // for some reason core doesn't set this field right
+            for input in &mut tx.input {
+                input.sequence = 0xFFFFFFFD;
+            }
+        }
+
+        let tx = self
+            .sign_raw_transaction_with_wallet(&serialize(&tx), None, None)
+            .unwrap();
+
+        // broadcast through electrum so that it caches the tx immediately
+        let txid = self
+            .electrum
+            .transaction_broadcast(&deserialize(&tx.hex).unwrap())
+            .unwrap();
+
+        if let Some(num) = meta_tx.min_confirmations {
+            self.generate(num, None);
+        }
+
+        let monitor_script = Address::from_str(&meta_tx.output[0].to_address)
+            .unwrap()
+            .script_pubkey();
+        self.wait_for_tx(txid, &monitor_script);
+
+        debug!("Sent tx: {}", txid);
+
+        txid
+    }
+
+    pub fn bump_fee(&mut self, txid: &Txid) -> Txid {
+        let tx = self.get_raw_transaction_info(txid, None).unwrap();
+        assert!(
+            tx.confirmations.is_none(),
+            "Can't bump tx {} because it's already confirmed",
+            txid
+        );
+
+        let bumped: serde_json::Value = self.call("bumpfee", &[txid.to_string().into()]).unwrap();
+        let new_txid = Txid::from_str(&bumped["txid"].as_str().unwrap().to_string()).unwrap();
+
+        let monitor_script =
+            tx.vout[0].script_pub_key.addresses.as_ref().unwrap()[0].script_pubkey();
+        self.wait_for_tx(new_txid, &monitor_script);
+
+        debug!("Bumped {}, new txid {}", txid, new_txid);
+
+        new_txid
+    }
+
+    pub fn generate_manually(&mut self, txs: Vec<Transaction>) -> String {
+        use bitcoin::blockdata::block::{Block, BlockHeader};
+        use bitcoin::blockdata::script::Builder;
+        use bitcoin::blockdata::transaction::{OutPoint, TxIn, TxOut};
+        use bitcoin::hash_types::{BlockHash, TxMerkleNode};
+
+        let block_template: serde_json::Value = self
+            .call("getblocktemplate", &[json!({"rules": ["segwit"]})])
+            .unwrap();
+        trace!("getblocktemplate: {:#?}", block_template);
+
+        let header = BlockHeader {
+            version: block_template["version"].as_i64().unwrap() as i32,
+            prev_blockhash: BlockHash::from_hex(
+                block_template["previousblockhash"].as_str().unwrap(),
+            )
+            .unwrap(),
+            merkle_root: TxMerkleNode::default(),
+            time: block_template["curtime"].as_u64().unwrap() as u32,
+            bits: u32::from_str_radix(block_template["bits"].as_str().unwrap(), 16).unwrap(),
+            nonce: 0,
+        };
+        debug!("header: {:#?}", header);
+
+        let height = block_template["height"].as_u64().unwrap() as i64;
+        let witness_reserved_value: Vec<u8> = sha256d::Hash::default().as_ref().into();
+        // burn block subsidy and fees, not a big deal
+        let mut coinbase_tx = Transaction {
+            version: 1,
+            lock_time: 0,
+            input: vec![TxIn {
+                previous_output: OutPoint::null(),
+                script_sig: Builder::new().push_int(height).into_script(),
+                sequence: 0xFFFFFFFF,
+                witness: vec![witness_reserved_value],
+            }],
+            output: vec![],
+        };
+
+        let mut txdata = vec![coinbase_tx.clone()];
+        txdata.extend_from_slice(&txs);
+
+        let mut block = Block { header, txdata };
+
+        let witness_root = block.witness_root();
+        let witness_commitment =
+            Block::compute_witness_commitment(&witness_root, &coinbase_tx.input[0].witness[0]);
+
+        // now update and replace the coinbase tx
+        let mut coinbase_witness_commitment_script = vec![0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed];
+        coinbase_witness_commitment_script.extend_from_slice(&witness_commitment);
+
+        coinbase_tx.output.push(TxOut {
+            value: 0,
+            script_pubkey: coinbase_witness_commitment_script.into(),
+        });
+        block.txdata[0] = coinbase_tx;
+
+        // set merkle root
+        let merkle_root = block.merkle_root();
+        block.header.merkle_root = merkle_root;
+
+        assert!(block.check_merkle_root());
+        assert!(block.check_witness_commitment());
+
+        // now do PoW :)
+        let target = block.header.target();
+        while block.header.validate_pow(&target).is_err() {
+            block.header.nonce = block.header.nonce.checked_add(1).unwrap(); // panic if we run out of nonces
+        }
+
+        let block_hex: String = serialize(&block).to_hex();
+        debug!("generated block hex: {}", block_hex);
+
+        self.electrum.block_headers_subscribe().unwrap();
+
+        let submit_result: serde_json::Value =
+            self.call("submitblock", &[block_hex.into()]).unwrap();
+        debug!("submitblock: {:?}", submit_result);
+        assert!(
+            submit_result.is_null(),
+            "submitblock error: {:?}",
+            submit_result.as_str()
+        );
+
+        self.wait_for_block(height as usize);
+
+        block.header.block_hash().to_hex()
+    }
+
+    pub fn generate(&mut self, num_blocks: u64, address: Option<Address>) {
+        let address = address.unwrap_or_else(|| self.get_new_address(None, None).unwrap());
+        let hashes = self.generate_to_address(num_blocks, &address).unwrap();
+        let best_hash = hashes.last().unwrap();
+        let height = self.get_block_info(best_hash).unwrap().height;
+
+        self.wait_for_block(height);
+
+        debug!("Generated blocks to new height {}", height);
+    }
+
+    pub fn invalidate(&mut self, num_blocks: u64) {
+        self.electrum.block_headers_subscribe().unwrap();
+
+        let best_hash = self.get_best_block_hash().unwrap();
+        let initial_height = self.get_block_info(&best_hash).unwrap().height;
+
+        let mut to_invalidate = best_hash;
+        for i in 1..=num_blocks {
+            trace!(
+                "Invalidating block {}/{} ({})",
+                i,
+                num_blocks,
+                to_invalidate
+            );
+
+            self.invalidate_block(&to_invalidate).unwrap();
+            to_invalidate = self.get_best_block_hash().unwrap();
+        }
+
+        self.wait_for_block(initial_height - num_blocks as usize);
+
+        debug!(
+            "Invalidated {} blocks to new height of {}",
+            num_blocks,
+            initial_height - num_blocks as usize
+        );
+    }
+
+    pub fn reorg(&mut self, num_blocks: u64) {
+        self.invalidate(num_blocks);
+        self.generate(num_blocks, None);
+    }
+
+    pub fn get_node_address(&self, address_type: Option<AddressType>) -> Address {
+        Address::from_str(
+            &self
+                .get_new_address(None, address_type)
+                .unwrap()
+                .to_string(),
+        )
+        .unwrap()
+    }
+}
+
+pub fn get_electrum_url() -> String {
+    env::var("BDK_ELECTRUM_URL").unwrap_or_else(|_| "tcp://127.0.0.1:50001".to_string())
+}
+
+impl Deref for TestClient {
+    type Target = RpcClient;
+
+    fn deref(&self) -> &Self::Target {
+        &self.client
+    }
+}
+
+impl Default for TestClient {
+    fn default() -> Self {
+        let rpc_host_and_port =
+            env::var("BDK_RPC_URL").unwrap_or_else(|_| "127.0.0.1:18443".to_string());
+        let wallet = env::var("BDK_RPC_WALLET").unwrap_or_else(|_| "bdk-test".to_string());
+        Self::new(rpc_host_and_port, wallet)
+    }
+}
+
+fn exponential_backoff_poll<T, F>(mut poll: F) -> T
+where
+    F: FnMut() -> Option<T>,
+{
+    let mut delay = Duration::from_millis(64);
+    loop {
+        match poll() {
+            Some(data) => break data,
+            None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0),
+            None => {}
+        }
+
+        std::thread::sleep(delay);
+    }
+}
+
+// TODO: we currently only support env vars, we could also parse a toml file
+fn get_auth() -> Auth {
+    match env::var("BDK_RPC_AUTH").as_ref().map(String::as_ref) {
+        Ok("USER_PASS") => Auth::UserPass(
+            env::var("BDK_RPC_USER").unwrap(),
+            env::var("BDK_RPC_PASS").unwrap(),
+        ),
+        _ => Auth::CookieFile(PathBuf::from(
+            env::var("BDK_RPC_COOKIEFILE")
+                .unwrap_or_else(|_| "/home/user/.bitcoin/regtest/.cookie".to_string()),
+        )),
+    }
+}
+
+/// This macro runs blockchain tests against a `Blockchain` implementation. It requires access to a
+/// Bitcoin core wallet via RPC. At the moment you have to dig into the code yourself and look at
+/// the setup required to run the tests yourself.
+#[macro_export]
+macro_rules! bdk_blockchain_tests {
+    (
+     fn test_instance() -> $blockchain:ty $block:block) => {
+        #[cfg(test)]
+        mod bdk_blockchain_tests {
+            use $crate::bitcoin::Network;
+            use $crate::testutils::blockchain_tests::TestClient;
+            use $crate::blockchain::noop_progress;
+            use $crate::database::MemoryDatabase;
+            use $crate::types::KeychainKind;
+            use $crate::{Wallet, FeeRate};
+            use $crate::testutils;
+            use $crate::serial_test::serial;
+
+            use super::*;
+
+            fn get_blockchain() -> $blockchain {
+                $block
+            }
+
+            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<$blockchain, MemoryDatabase> {
+                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
+            }
+
+            fn init_single_sig() -> (Wallet<$blockchain, MemoryDatabase>, (String, Option<String>), TestClient) {
+                let _ = env_logger::try_init();
+
+                let descriptors = testutils! {
+                    @descriptors ( "wpkh(Alice)" ) ( "wpkh(Alice)" ) ( @keys ( "Alice" => (@generate_xprv "/44'/0'/0'/0/*", "/44'/0'/0'/1/*") ) )
+                };
+
+                let test_client = TestClient::default();
+                let wallet = get_wallet_from_descriptors(&descriptors);
+
+                // rpc need to call import_multi before receiving any tx, otherwise will not see tx in the mempool
+                #[cfg(feature = "test-rpc")]
+                wallet.sync(noop_progress(), None).unwrap();
+
+                (wallet, descriptors, test_client)
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_simple() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let tx = testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                };
+                println!("{:?}", tx);
+                let txid = test_client.receive(tx);
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External, "incorrect keychain kind");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_stop_gap_20() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 5) => 50_000 )
+                });
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 25) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 100_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_before_and_after_receive() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 0);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_multiple_outputs_same_tx() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000, (@external descriptors, 5) => 30_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 105_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 3, "incorrect number of unspents");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 105_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_receive_multi() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 5) => 25_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 2, "incorrect number of unspent");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_address_reuse() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 25_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_receive_rbf_replaced() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) ( @replaceable true )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation_time");
+
+                let new_txid = test_client.bump_fee(&txid);
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after bump");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs after bump");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent after bump");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, new_txid, "incorrect txid after bump");
+                assert_eq!(list_tx_item.received, 50_000, "incorrect received after bump");
+                assert_eq!(list_tx_item.sent, 0, "incorrect sent after bump");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect height after bump");
+            }
+
+            // FIXME: I would like this to be cfg_attr(not(feature = "test-esplora"), ignore) but it
+            // doesn't work for some reason.
+            #[cfg(not(feature = "esplora"))]
+            #[test]
+            #[serial]
+            fn test_sync_reorg_block() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                let txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) ( @confirmations 1 ) ( @replaceable true )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid");
+                assert!(list_tx_item.confirmation_time.is_some(), "incorrect confirmation_time");
+
+                // Invalidate 1 block
+                test_client.invalidate(1);
+
+                wallet.sync(noop_progress(), None).unwrap();
+
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after invalidate");
+
+                let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                assert_eq!(list_tx_item.txid, txid, "incorrect txid after invalidate");
+                assert_eq!(list_tx_item.confirmation_time, None, "incorrect confirmation time after invalidate");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_after_send() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                println!("{}", descriptors.0);
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey(), 25_000);
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                let tx = psbt.extract_tx();
+                println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
+                wallet.broadcast(tx).unwrap();
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after send");
+
+                assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs");
+                assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents");
+            }
+
+            #[test]
+            #[serial]
+            fn test_update_confirmation_time_after_generate() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                println!("{}", descriptors.0);
+                let node_addr = test_client.get_node_address(None);
+
+                let received_txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_none());
+
+                test_client.generate(1, Some(node_addr));
+                wallet.sync(noop_progress(), None).unwrap();
+
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+                let details = tx_map.get(&received_txid).unwrap();
+                assert!(details.confirmation_time.is_some());
+
+            }
+
+
+            #[test]
+            #[serial]
+            fn test_sync_outgoing_from_scratch() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+                let received_txid = test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey(), 25_000);
+                let (mut psbt, details) = builder.finish().unwrap();
+
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after receive");
+
+                // empty wallet
+                let wallet = get_wallet_from_descriptors(&descriptors);
+
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
+                wallet.sync(noop_progress(), None).unwrap();
+                let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+
+                let received = tx_map.get(&received_txid).unwrap();
+                assert_eq!(received.received, 50_000, "incorrect received from receiver");
+                assert_eq!(received.sent, 0, "incorrect sent from receiver");
+
+                let sent = tx_map.get(&sent_txid).unwrap();
+                assert_eq!(sent.received, details.received, "incorrect received from sender");
+                assert_eq!(sent.sent, details.sent, "incorrect sent from sender");
+                assert_eq!(sent.fee.unwrap_or(0), details.fee.unwrap_or(0), "incorrect fees from sender");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_long_change_chain() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 )
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut total_sent = 0;
+                for _ in 0..5 {
+                    let mut builder = wallet.build_tx();
+                    builder.add_recipient(node_addr.script_pubkey(), 5_000);
+                    let (mut psbt, details) = builder.finish().unwrap();
+                    let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    total_sent += 5_000 + details.fee.unwrap_or(0);
+                }
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance after chain");
+
+                // empty wallet
+
+                let wallet = get_wallet_from_descriptors(&descriptors);
+
+                #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
+                test_client.generate(1, Some(node_addr));
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance empty wallet");
+
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_bump_fee_basic() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 5_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance from received");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(2.1));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fee.unwrap_or(0) - 5_000, "incorrect balance from fees after bump");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance from received after bump");
+
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_bump_fee_remove_change() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect received after send");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(5.0));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after change removal");
+                assert_eq!(new_details.received, 0, "incorrect received after change removal");
+
+                assert!(new_details.fee.unwrap_or(0) > details.fee.unwrap_or(0), "incorrect fees");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_bump_fee_add_input_simple() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(10.0));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance after add input");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_bump_fee_add_input_no_change() {
+                let (wallet, descriptors, mut test_client) = init_single_sig();
+                let node_addr = test_client.get_node_address(None);
+
+                test_client.receive(testutils! {
+                    @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
+                });
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance");
+
+                let mut builder = wallet.build_tx();
+                builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf();
+                let (mut psbt, details) = builder.finish().unwrap();
+                let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fee.unwrap_or(0), "incorrect balance after send");
+                assert_eq!(details.received, 1_000 - details.fee.unwrap_or(0), "incorrect received after send");
+
+                let mut builder = wallet.build_fee_bump(details.txid).unwrap();
+                builder.fee_rate(FeeRate::from_sat_per_vb(123.0));
+                let (mut new_psbt, new_details) = builder.finish().unwrap();
+                println!("{:#?}", new_details);
+
+                let finalized = wallet.sign(&mut new_psbt, Default::default()).unwrap();
+                assert!(finalized, "Cannot finalize transaction");
+                wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(new_details.sent, 75_000, "incorrect sent");
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after add input");
+                assert_eq!(new_details.received, 0, "incorrect received after add input");
+            }
+
+            #[test]
+            #[serial]
+            fn test_sync_receive_coinbase() {
+                let (wallet, _, mut test_client) = init_single_sig();
+
+                let wallet_addr = wallet.get_address($crate::wallet::AddressIndex::New).unwrap().address;
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance");
+
+                test_client.generate(1, Some(wallet_addr));
+
+                #[cfg(feature = "rpc")]
+                {
+                    // rpc consider coinbase only when mature (100 blocks)
+                    let node_addr = test_client.get_node_address(None);
+                    test_client.generate(100, Some(node_addr));
+                }
+
+
+                wallet.sync(noop_progress(), None).unwrap();
+                assert!(wallet.get_balance().unwrap() > 0, "incorrect balance after receiving coinbase");
+            }
+        }
+    }
+}

src/testutils/mod.rs

@@ -0,0 +1,230 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+#![allow(missing_docs)]
+
+#[cfg(feature = "test-blockchains")]
+pub mod blockchain_tests;
+
+use bitcoin::secp256k1::{Secp256k1, Verification};
+use bitcoin::{Address, PublicKey};
+
+use miniscript::descriptor::DescriptorPublicKey;
+use miniscript::{Descriptor, MiniscriptKey, TranslatePk};
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingOutput {
+    pub value: u64,
+    pub to_address: String,
+}
+
+impl TestIncomingOutput {
+    pub fn new(value: u64, to_address: Address) -> Self {
+        Self {
+            value,
+            to_address: to_address.to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingTx {
+    pub output: Vec<TestIncomingOutput>,
+    pub min_confirmations: Option<u64>,
+    pub locktime: Option<i64>,
+    pub replaceable: Option<bool>,
+}
+
+impl TestIncomingTx {
+    pub fn new(
+        output: Vec<TestIncomingOutput>,
+        min_confirmations: Option<u64>,
+        locktime: Option<i64>,
+        replaceable: Option<bool>,
+    ) -> Self {
+        Self {
+            output,
+            min_confirmations,
+            locktime,
+            replaceable,
+        }
+    }
+
+    pub fn add_output(&mut self, output: TestIncomingOutput) {
+        self.output.push(output);
+    }
+}
+
+#[doc(hidden)]
+pub trait TranslateDescriptor {
+    // derive and translate a `Descriptor<DescriptorPublicKey>` into a `Descriptor<PublicKey>`
+    fn derive_translated<C: Verification>(
+        &self,
+        secp: &Secp256k1<C>,
+        index: u32,
+    ) -> Descriptor<PublicKey>;
+}
+
+impl TranslateDescriptor for Descriptor<DescriptorPublicKey> {
+    fn derive_translated<C: Verification>(
+        &self,
+        secp: &Secp256k1<C>,
+        index: u32,
+    ) -> Descriptor<PublicKey> {
+        let translate = |key: &DescriptorPublicKey| -> PublicKey {
+            match key {
+                DescriptorPublicKey::XPub(xpub) => {
+                    xpub.xkey
+                        .derive_pub(secp, &xpub.derivation_path)
+                        .expect("hardened derivation steps")
+                        .public_key
+                }
+                DescriptorPublicKey::SinglePub(key) => key.key,
+            }
+        };
+
+        self.derive(index)
+            .translate_pk_infallible(|pk| translate(pk), |pkh| translate(pkh).to_pubkeyhash())
+    }
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! testutils {
+    ( @external $descriptors:expr, $child:expr ) => ({
+        use bitcoin::secp256k1::Secp256k1;
+        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
+
+        use $crate::testutils::TranslateDescriptor;
+
+        let secp = Secp256k1::new();
+
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &$descriptors.0).expect("Failed to parse descriptor in `testutils!(@external)`").0;
+        parsed.derive_translated(&secp, $child).address(bitcoin::Network::Regtest).expect("No address form")
+    });
+    ( @internal $descriptors:expr, $child:expr ) => ({
+        use bitcoin::secp256k1::Secp256k1;
+        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorTrait};
+
+        use $crate::testutils::TranslateDescriptor;
+
+        let secp = Secp256k1::new();
+
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
+        parsed.derive_translated(&secp, $child).address(bitcoin::Network::Regtest).expect("No address form")
+    });
+    ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
+    ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
+
+    ( @tx ( $( ( $( $addr:tt )* ) => $amount:expr ),+ ) $( ( @locktime $locktime:expr ) )? $( ( @confirmations $confirmations:expr ) )? $( ( @replaceable $replaceable:expr ) )? ) => ({
+        let outs = vec![$( $crate::testutils::TestIncomingOutput::new($amount, testutils!( $($addr)* ))),+];
+
+        let locktime = None::<i64>$(.or(Some($locktime)))?;
+
+        let min_confirmations = None::<u64>$(.or(Some($confirmations)))?;
+        let replaceable = None::<bool>$(.or(Some($replaceable)))?;
+
+        $crate::testutils::TestIncomingTx::new(outs, min_confirmations, locktime, replaceable)
+    });
+
+    ( @literal $key:expr ) => ({
+        let key = $key.to_string();
+        (key, None::<String>, None::<String>)
+    });
+    ( @generate_xprv $( $external_path:expr )? $( ,$internal_path:expr )? ) => ({
+        use rand::Rng;
+
+        let mut seed = [0u8; 32];
+        rand::thread_rng().fill(&mut seed[..]);
+
+        let key = bitcoin::util::bip32::ExtendedPrivKey::new_master(
+            bitcoin::Network::Testnet,
+            &seed,
+        );
+
+        let external_path = None::<String>$(.or(Some($external_path.to_string())))?;
+        let internal_path = None::<String>$(.or(Some($internal_path.to_string())))?;
+
+        (key.unwrap().to_string(), external_path, internal_path)
+    });
+    ( @generate_wif ) => ({
+        use rand::Rng;
+
+        let mut key = [0u8; bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
+        rand::thread_rng().fill(&mut key[..]);
+
+        (bitcoin::PrivateKey {
+            compressed: true,
+            network: bitcoin::Network::Testnet,
+            key: bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
+        }.to_string(), None::<String>, None::<String>)
+    });
+
+    ( @keys ( $( $alias:expr => ( $( $key_type:tt )* ) ),+ ) ) => ({
+        let mut map = std::collections::HashMap::new();
+        $(
+            let alias: &str = $alias;
+            map.insert(alias, testutils!( $($key_type)* ));
+        )+
+
+        map
+    });
+
+    ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )? $( ( @keys $( $keys:tt )* ) )* ) => ({
+        use std::str::FromStr;
+        use std::collections::HashMap;
+        use miniscript::descriptor::Descriptor;
+        use miniscript::TranslatePk;
+
+        #[allow(unused_assignments, unused_mut)]
+        let mut keys: HashMap<&'static str, (String, Option<String>, Option<String>)> = HashMap::new();
+        $(
+            keys = testutils!{ @keys $( $keys )* };
+        )*
+
+        let external: Descriptor<String> = FromStr::from_str($external_descriptor).unwrap();
+        let external: Descriptor<String> = external.translate_pk_infallible::<_, _>(|k| {
+            if let Some((key, ext_path, _)) = keys.get(&k.as_str()) {
+                format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))
+            } else {
+                k.clone()
+            }
+        }, |kh| {
+            if let Some((key, ext_path, _)) = keys.get(&kh.as_str()) {
+                format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))
+            } else {
+                kh.clone()
+            }
+
+        });
+        let external = external.to_string();
+
+        let internal = None::<String>$(.or({
+            let string_internal: Descriptor<String> = FromStr::from_str($internal_descriptor).unwrap();
+
+            let string_internal: Descriptor<String> = string_internal.translate_pk_infallible::<_, _>(|k| {
+                if let Some((key, _, int_path)) = keys.get(&k.as_str()) {
+                    format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))
+                } else {
+                    k.clone()
+                }
+            }, |kh| {
+                if let Some((key, _, int_path)) = keys.get(&kh.as_str()) {
+                    format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))
+                } else {
+                    kh.clone()
+                }
+            });
+            Some(string_internal.to_string())
+        }))?;
+
+        (external, internal)
+    })
+}

src/types.rs

@@ -1,59 +1,45 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 use std::convert::AsRef;
 
 use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
-use bitcoin::hash_types::Txid;
+use bitcoin::{hash_types::Txid, util::psbt};
 
 use serde::{Deserialize, Serialize};
 
-/// Types of script
+/// Types of keychains
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum ScriptType {
+pub enum KeychainKind {
+    /// External
     External = 0,
+    /// Internal, usually used for change outputs
     Internal = 1,
 }
 
-impl ScriptType {
+impl KeychainKind {
+    /// Return [`KeychainKind`] as a byte
     pub fn as_byte(&self) -> u8 {
         match self {
-            ScriptType::External => 'e' as u8,
-            ScriptType::Internal => 'i' as u8,
+            KeychainKind::External => b'e',
+            KeychainKind::Internal => b'i',
         }
     }
-
-    pub fn is_internal(&self) -> bool {
-        self == &ScriptType::Internal
-    }
 }
 
-impl AsRef<[u8]> for ScriptType {
+impl AsRef<[u8]> for KeychainKind {
     fn as_ref(&self) -> &[u8] {
         match self {
-            ScriptType::External => b"e",
-            ScriptType::Internal => b"i",
+            KeychainKind::External => b"e",
+            KeychainKind::Internal => b"i",
         }
     }
 }
@@ -70,12 +56,12 @@ impl FeeRate {
     }
 
     /// Create a new instance of [`FeeRate`] given a float fee rate in satoshi/vbyte
-    pub fn from_sat_per_vb(sat_per_vb: f32) -> Self {
+    pub const fn from_sat_per_vb(sat_per_vb: f32) -> Self {
         FeeRate(sat_per_vb)
     }
 
     /// Create a new [`FeeRate`] with the default min relay fee value
-    pub fn default_min_relay_fee() -> Self {
+    pub const fn default_min_relay_fee() -> Self {
         FeeRate(1.0)
     }
 
@@ -91,22 +77,131 @@ impl std::default::Default for FeeRate {
     }
 }
 
-/// A wallet unspent output
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub struct UTXO {
+/// An unspent output owned by a [`Wallet`].
+///
+/// [`Wallet`]: crate::Wallet
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LocalUtxo {
+    /// Reference to a transaction output
     pub outpoint: OutPoint,
+    /// Transaction output
     pub txout: TxOut,
-    pub is_internal: bool,
+    /// Type of keychain
+    pub keychain: KeychainKind,
+}
+
+/// A [`Utxo`] with its `satisfaction_weight`.
+#[derive(Debug, Clone, PartialEq)]
+pub struct WeightedUtxo {
+    /// The weight of the witness data and `scriptSig` expressed in [weight units]. This is used to
+    /// properly maintain the feerate when adding this input to a transaction during coin selection.
+    ///
+    /// [weight units]: https://en.bitcoin.it/wiki/Weight_units
+    pub satisfaction_weight: usize,
+    /// The UTXO
+    pub utxo: Utxo,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+/// An unspent transaction output (UTXO).
+pub enum Utxo {
+    /// A UTXO owned by the local wallet.
+    Local(LocalUtxo),
+    /// A UTXO owned by another wallet.
+    Foreign {
+        /// The location of the output.
+        outpoint: OutPoint,
+        /// The information about the input we require to add it to a PSBT.
+        // Box it to stop the type being too big.
+        psbt_input: Box<psbt::Input>,
+    },
+}
+
+impl Utxo {
+    /// Get the location of the UTXO
+    pub fn outpoint(&self) -> OutPoint {
+        match &self {
+            Utxo::Local(local) => local.outpoint,
+            Utxo::Foreign { outpoint, .. } => *outpoint,
+        }
+    }
+
+    /// Get the `TxOut` of the UTXO
+    pub fn txout(&self) -> &TxOut {
+        match &self {
+            Utxo::Local(local) => &local.txout,
+            Utxo::Foreign {
+                outpoint,
+                psbt_input,
+            } => {
+                if let Some(prev_tx) = &psbt_input.non_witness_utxo {
+                    return &prev_tx.output[outpoint.vout as usize];
+                }
+
+                if let Some(txout) = &psbt_input.witness_utxo {
+                    return txout;
+                }
+
+                unreachable!("Foreign UTXOs will always have one of these set")
+            }
+        }
+    }
 }
 
 /// A wallet transaction
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
 pub struct TransactionDetails {
+    /// Optional transaction
     pub transaction: Option<Transaction>,
+    /// Transaction id
     pub txid: Txid,
-    pub timestamp: u64,
+
+    /// Received value (sats)
     pub received: u64,
+    /// Sent value (sats)
     pub sent: u64,
-    pub fees: u64,
-    pub height: Option<u32>,
+    /// Fee value (sats) if available
+    pub fee: Option<u64>,
+    /// If the transaction is confirmed, contains height and timestamp of the block containing the
+    /// transaction, unconfirmed transaction contains `None`.
+    pub confirmation_time: Option<ConfirmationTime>,
+    /// Whether the tx has been verified against the consensus rules
+    ///
+    /// Confirmed txs are considered "verified" by default, while unconfirmed txs are checked to
+    /// ensure an unstrusted [`Blockchain`](crate::blockchain::Blockchain) backend can't trick the
+    /// wallet into using an invalid tx as an RBF template.
+    ///
+    /// The check is only perfomed when the `verify` feature is enabled.
+    #[serde(default = "bool::default")] // default to `false` if not specified
+    pub verified: bool,
+}
+
+/// Block height and timestamp of the block containing the confirmed transaction
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
+pub struct ConfirmationTime {
+    /// confirmation block height
+    pub height: u32,
+    /// confirmation block timestamp
+    pub timestamp: u64,
+}
+
+impl ConfirmationTime {
+    /// Returns `Some` `ConfirmationTime` if both `height` and `timestamp` are `Some`
+    pub fn new(height: Option<u32>, timestamp: Option<u64>) -> Option<Self> {
+        match (height, timestamp) {
+            (Some(height), Some(timestamp)) => Some(ConfirmationTime { height, timestamp }),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn can_store_feerate_in_const() {
+        const _MY_RATE: FeeRate = FeeRate::from_sat_per_vb(10.0);
+        const _MIN_RELAY: FeeRate = FeeRate::default_min_relay_fee();
+    }
 }

src/wallet/address_validator.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Address validation callbacks
 //!
@@ -33,7 +20,7 @@
 //! An address validator can be attached to a [`Wallet`](super::Wallet) by using the
 //! [`Wallet::add_address_validator`](super::Wallet::add_address_validator) method, and
 //! whenever a new address is generated (either explicitly by the user with
-//! [`Wallet::get_new_address`](super::Wallet::get_new_address) or internally to create a change
+//! [`Wallet::get_address`](super::Wallet::get_address) or internally to create a change
 //! address) all the attached validators will be polled, in sequence. All of them must complete
 //! successfully to continue.
 //!
@@ -42,23 +29,25 @@
 //! ```
 //! # use std::sync::Arc;
 //! # use bitcoin::*;
-//! # use magical::address_validator::*;
-//! # use magical::database::*;
-//! # use magical::*;
+//! # use bdk::address_validator::*;
+//! # use bdk::database::*;
+//! # use bdk::*;
+//! # use bdk::wallet::AddressIndex::New;
+//! #[derive(Debug)]
 //! struct PrintAddressAndContinue;
 //!
 //! impl AddressValidator for PrintAddressAndContinue {
 //!     fn validate(
 //!         &self,
-//!         script_type: ScriptType,
-//!         hd_keypaths: &HDKeyPaths,
+//!         keychain: KeychainKind,
+//!         hd_keypaths: &HdKeyPaths,
 //!         script: &Script
 //!     ) -> Result<(), AddressValidatorError> {
 //!         let address = Address::from_script(script, Network::Testnet)
 //!             .as_ref()
 //!             .map(Address::to_string)
 //!             .unwrap_or(script.to_string());
-//!         println!("New address of type {:?}: {}", script_type, address);
+//!         println!("New address of type {:?}: {}", keychain, address);
 //!         println!("HD keypaths: {:#?}", hd_keypaths);
 //!
 //!         Ok(())
@@ -66,28 +55,33 @@
 //! }
 //!
 //! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)";
-//! let mut wallet: OfflineWallet<_> = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
-//! wallet.add_address_validator(Arc::new(Box::new(PrintAddressAndContinue)));
+//! let mut wallet = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
+//! wallet.add_address_validator(Arc::new(PrintAddressAndContinue));
 //!
-//! let address = wallet.get_new_address()?;
+//! let address = wallet.get_address(New)?;
 //! println!("Address: {}", address);
-//! # Ok::<(), magical::Error>(())
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::fmt;
 
 use bitcoin::Script;
 
-use crate::descriptor::HDKeyPaths;
-use crate::types::ScriptType;
+use crate::descriptor::HdKeyPaths;
+use crate::types::KeychainKind;
 
 /// Errors that can be returned to fail the validation of an address
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum AddressValidatorError {
+    /// User rejected the address
     UserRejected,
+    /// Network connection error
     ConnectionError,
+    /// Network request timeout error
     TimeoutError,
+    /// Invalid script
     InvalidScript,
+    /// A custom error message
     Message(String),
 }
 
@@ -106,12 +100,12 @@ impl std::error::Error for AddressValidatorError {}
 /// validator will be propagated up to the original caller that triggered the address generation.
 ///
 /// For a usage example see [this module](crate::address_validator)'s documentation.
-pub trait AddressValidator {
+pub trait AddressValidator: Send + Sync + fmt::Debug {
     /// Validate or inspect an address
     fn validate(
         &self,
-        script_type: ScriptType,
-        hd_keypaths: &HDKeyPaths,
+        keychain: KeychainKind,
+        hd_keypaths: &HdKeyPaths,
         script: &Script,
     ) -> Result<(), AddressValidatorError>;
 }
@@ -122,14 +116,15 @@ mod test {
 
     use super::*;
     use crate::wallet::test::{get_funded_wallet, get_test_wpkh};
-    use crate::wallet::TxBuilder;
+    use crate::wallet::AddressIndex::New;
 
+    #[derive(Debug)]
     struct TestValidator;
     impl AddressValidator for TestValidator {
         fn validate(
             &self,
-            _script_type: ScriptType,
-            _hd_keypaths: &HDKeyPaths,
+            _keychain: KeychainKind,
+            _hd_keypaths: &HdKeyPaths,
             _script: &bitcoin::Script,
         ) -> Result<(), AddressValidatorError> {
             Err(AddressValidatorError::InvalidScript)
@@ -140,23 +135,20 @@ mod test {
     #[should_panic(expected = "InvalidScript")]
     fn test_address_validator_external() {
         let (mut wallet, _, _) = get_funded_wallet(get_test_wpkh());
-        wallet.add_address_validator(Arc::new(Box::new(TestValidator)));
+        wallet.add_address_validator(Arc::new(TestValidator));
 
-        wallet.get_new_address().unwrap();
+        wallet.get_address(New).unwrap();
     }
 
     #[test]
     #[should_panic(expected = "InvalidScript")]
     fn test_address_validator_internal() {
         let (mut wallet, descriptors, _) = get_funded_wallet(get_test_wpkh());
-        wallet.add_address_validator(Arc::new(Box::new(TestValidator)));
+        wallet.add_address_validator(Arc::new(TestValidator));
 
-        let addr = testutils!(@external descriptors, 10);
-        wallet
-            .create_tx(TxBuilder::with_recipients(vec![(
-                addr.script_pubkey(),
-                25_000,
-            )]))
-            .unwrap();
+        let addr = crate::testutils!(@external descriptors, 10);
+        let mut builder = wallet.build_tx();
+        builder.add_recipient(addr.script_pubkey(), 25_000);
+        builder.finish().unwrap();
     }
 }

src/wallet/export.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Wallet export
 //!
@@ -33,9 +20,9 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use magical::database::*;
-//! # use magical::wallet::export::*;
-//! # use magical::*;
+//! # use bdk::database::*;
+//! # use bdk::wallet::export::*;
+//! # use bdk::*;
 //! let import = r#"{
 //!     "descriptor": "wpkh([c258d2e4\/84h\/1h\/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe\/0\/*)",
 //!     "blockheight":1782088,
@@ -43,17 +30,22 @@
 //! }"#;
 //!
 //! let import = WalletExport::from_str(import)?;
-//! let wallet: OfflineWallet<_> = Wallet::new_offline(&import.descriptor(), import.change_descriptor().as_deref(), Network::Testnet, MemoryDatabase::default())?;
-//! # Ok::<_, magical::Error>(())
+//! let wallet = Wallet::new_offline(
+//!     &import.descriptor(),
+//!     import.change_descriptor().as_ref(),
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//! )?;
+//! # Ok::<_, bdk::Error>(())
 //! ```
 //!
 //! ### Export a `Wallet`
 //! ```
 //! # use bitcoin::*;
-//! # use magical::database::*;
-//! # use magical::wallet::export::*;
-//! # use magical::*;
-//! let wallet: OfflineWallet<_> = Wallet::new_offline(
+//! # use bdk::database::*;
+//! # use bdk::wallet::export::*;
+//! # use bdk::*;
+//! let wallet = Wallet::new_offline(
 //!     "wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/0/*)",
 //!     Some("wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/1/*)"),
 //!     Network::Testnet,
@@ -61,19 +53,19 @@
 //! )?;
 //! let export = WalletExport::export_wallet(&wallet, "exported wallet", true)
 //!     .map_err(ToString::to_string)
-//!     .map_err(magical::Error::Generic)?;
+//!     .map_err(bdk::Error::Generic)?;
 //!
 //! println!("Exported: {}", export.to_string());
-//! # Ok::<_, magical::Error>(())
+//! # Ok::<_, bdk::Error>(())
 //! ```
 
 use std::str::FromStr;
 
 use serde::{Deserialize, Serialize};
 
-use miniscript::{Descriptor, ScriptContext, Terminal};
+use miniscript::descriptor::{ShInner, WshInner};
+use miniscript::{Descriptor, DescriptorPublicKey, ScriptContext, Terminal};
 
-use crate::blockchain::Blockchain;
 use crate::database::BatchDatabase;
 use crate::wallet::Wallet;
 
@@ -103,6 +95,10 @@ impl FromStr for WalletExport {
     }
 }
 
+fn remove_checksum(s: String) -> String {
+    s.splitn(2, '#').next().map(String::from).unwrap()
+}
+
 impl WalletExport {
     /// Export a wallet
     ///
@@ -115,14 +111,15 @@ impl WalletExport {
     ///
     /// If the database is empty or `include_blockheight` is false, the `blockheight` field
     /// returned will be `0`.
-    pub fn export_wallet<B: Blockchain, D: BatchDatabase>(
+    pub fn export_wallet<B, D: BatchDatabase>(
         wallet: &Wallet<B, D>,
         label: &str,
         include_blockheight: bool,
     ) -> Result<Self, &'static str> {
         let descriptor = wallet
             .descriptor
-            .to_string_with_secret(&wallet.signers.as_key_map());
+            .to_string_with_secret(&wallet.signers.as_key_map(wallet.secp_ctx()));
+        let descriptor = remove_checksum(descriptor);
         Self::is_compatible_with_core(&descriptor)?;
 
         let blockheight = match wallet.database.borrow().iter_txs(false) {
@@ -131,9 +128,9 @@ impl WalletExport {
             Ok(txs) => {
                 let mut heights = txs
                     .into_iter()
-                    .map(|tx| tx.height.unwrap_or(0))
+                    .map(|tx| tx.confirmation_time.map(|c| c.height).unwrap_or(0))
                     .collect::<Vec<_>>();
-                heights.sort();
+                heights.sort_unstable();
 
                 *heights.last().unwrap_or(&0)
             }
@@ -145,12 +142,12 @@ impl WalletExport {
             blockheight,
         };
 
-        if export.change_descriptor()
-            != wallet
-                .change_descriptor
-                .as_ref()
-                .map(|d| d.to_string_with_secret(&wallet.change_signers.as_key_map()))
-        {
+        let desc_to_string = |d: &Descriptor<DescriptorPublicKey>| {
+            let descriptor =
+                d.to_string_with_secret(&wallet.change_signers.as_key_map(wallet.secp_ctx()));
+            remove_checksum(descriptor)
+        };
+        if export.change_descriptor() != wallet.change_descriptor.as_ref().map(desc_to_string) {
             return Err("Incompatible change descriptor");
         }
 
@@ -159,7 +156,7 @@ impl WalletExport {
 
     fn is_compatible_with_core(descriptor: &str) -> Result<(), &'static str> {
         fn check_ms<Ctx: ScriptContext>(
-            terminal: Terminal<String, Ctx>,
+            terminal: &Terminal<String, Ctx>,
         ) -> Result<(), &'static str> {
             if let Terminal::Multi(_, _) = terminal {
                 Ok(())
@@ -168,13 +165,22 @@ impl WalletExport {
             }
         }
 
+        // pkh(), wpkh(), sh(wpkh()) are always fine, as well as multi() and sortedmulti()
         match Descriptor::<String>::from_str(descriptor).map_err(|_| "Invalid descriptor")? {
-            Descriptor::Pk(_)
-            | Descriptor::Pkh(_)
-            | Descriptor::Wpkh(_)
-            | Descriptor::ShWpkh(_) => Ok(()),
-            Descriptor::Sh(ms) => check_ms(ms.node),
-            Descriptor::Wsh(ms) | Descriptor::ShWsh(ms) => check_ms(ms.node),
+            Descriptor::Pkh(_) | Descriptor::Wpkh(_) => Ok(()),
+            Descriptor::Sh(sh) => match sh.as_inner() {
+                ShInner::Wpkh(_) => Ok(()),
+                ShInner::SortedMulti(_) => Ok(()),
+                ShInner::Wsh(wsh) => match wsh.as_inner() {
+                    WshInner::SortedMulti(_) => Ok(()),
+                    WshInner::Ms(ms) => check_ms(&ms.node),
+                },
+                ShInner::Ms(ms) => check_ms(&ms.node),
+            },
+            Descriptor::Wsh(wsh) => match wsh.as_inner() {
+                WshInner::SortedMulti(_) => Ok(()),
+                WshInner::Ms(ms) => check_ms(&ms.node),
+            },
             _ => Err("The descriptor is not compatible with Bitcoin Core"),
         }
     }
@@ -205,7 +211,8 @@ mod test {
     use super::*;
     use crate::database::{memory::MemoryDatabase, BatchOperations};
     use crate::types::TransactionDetails;
-    use crate::wallet::{OfflineWallet, Wallet};
+    use crate::wallet::Wallet;
+    use crate::ConfirmationTime;
 
     fn get_test_db() -> MemoryDatabase {
         let mut db = MemoryDatabase::new();
@@ -215,11 +222,15 @@ mod test {
                 "4ddff1fa33af17f377f62b72357b43107c19110a8009b36fb832af505efed98a",
             )
             .unwrap(),
-            timestamp: 12345678,
+
             received: 100_000,
             sent: 0,
-            fees: 500,
-            height: Some(5000),
+            fee: Some(500),
+            confirmation_time: Some(ConfirmationTime {
+                timestamp: 12345678,
+                height: 5000,
+            }),
+            verified: true,
         })
         .unwrap();
 
@@ -231,10 +242,10 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
 
-        let wallet: OfflineWallet<_> = Wallet::new_offline(
+        let wallet = Wallet::new_offline(
             descriptor,
             Some(change_descriptor),
-            Network::Testnet,
+            Network::Bitcoin,
             get_test_db(),
         )
         .unwrap();
@@ -255,8 +266,8 @@ mod test {
 
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
 
-        let wallet: OfflineWallet<_> =
-            Wallet::new_offline(descriptor, None, Network::Testnet, get_test_db()).unwrap();
+        let wallet =
+            Wallet::new_offline(descriptor, None, Network::Bitcoin, get_test_db()).unwrap();
         WalletExport::export_wallet(&wallet, "Test Label", true).unwrap();
     }
 
@@ -269,10 +280,10 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/50'/0'/1/*)";
 
-        let wallet: OfflineWallet<_> = Wallet::new_offline(
+        let wallet = Wallet::new_offline(
             descriptor,
             Some(change_descriptor),
-            Network::Testnet,
+            Network::Bitcoin,
             get_test_db(),
         )
         .unwrap();
@@ -292,7 +303,7 @@ mod test {
                                        [c98b1535/48'/0'/0'/2']tpubDCDi5W4sP6zSnzJeowy8rQDVhBdRARaPhK1axABi8V1661wEPeanpEXj4ZLAUEoikVtoWcyK26TKKJSecSfeKxwHCcRrge9k1ybuiL71z4a/1/*\
                                  ))";
 
-        let wallet: OfflineWallet<_> = Wallet::new_offline(
+        let wallet = Wallet::new_offline(
             descriptor,
             Some(change_descriptor),
             Network::Testnet,
@@ -312,10 +323,10 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
 
-        let wallet: OfflineWallet<_> = Wallet::new_offline(
+        let wallet = Wallet::new_offline(
             descriptor,
             Some(change_descriptor),
-            Network::Testnet,
+            Network::Bitcoin,
             get_test_db(),
         )
         .unwrap();

src/wallet/rbf.rs

@@ -1,127 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-use crate::database::Database;
-use crate::error::Error;
-use crate::types::*;
-
-/// Filters unspent utxos
-pub(super) fn filter_available<I: Iterator<Item = UTXO>, D: Database>(
-    database: &D,
-    iter: I,
-) -> Result<Vec<UTXO>, Error> {
-    Ok(iter
-        .map(|utxo| {
-            Ok(match database.get_tx(&utxo.outpoint.txid, true)? {
-                None => None,
-                Some(tx) if tx.height.is_none() => None,
-                Some(_) => Some(utxo),
-            })
-        })
-        .collect::<Result<Vec<_>, Error>>()?
-        .into_iter()
-        .filter_map(|x| x)
-        .collect())
-}
-
-#[cfg(test)]
-mod test {
-    use std::str::FromStr;
-
-    use bitcoin::{OutPoint, Transaction, TxIn, TxOut, Txid};
-
-    use super::*;
-    use crate::database::{BatchOperations, MemoryDatabase};
-
-    fn add_transaction(
-        database: &mut MemoryDatabase,
-        spend: Vec<OutPoint>,
-        outputs: Vec<u64>,
-    ) -> Txid {
-        let tx = Transaction {
-            version: 1,
-            lock_time: 0,
-            input: spend
-                .iter()
-                .cloned()
-                .map(|previous_output| TxIn {
-                    previous_output,
-                    ..Default::default()
-                })
-                .collect(),
-            output: outputs
-                .iter()
-                .cloned()
-                .map(|value| TxOut {
-                    value,
-                    ..Default::default()
-                })
-                .collect(),
-        };
-        let txid = tx.txid();
-
-        for input in &spend {
-            database.del_utxo(input).unwrap();
-        }
-        for vout in 0..outputs.len() {
-            database
-                .set_utxo(&UTXO {
-                    txout: tx.output[vout].clone(),
-                    outpoint: OutPoint {
-                        txid,
-                        vout: vout as u32,
-                    },
-                    is_internal: true,
-                })
-                .unwrap();
-        }
-        database
-            .set_tx(&TransactionDetails {
-                txid,
-                transaction: Some(tx),
-                height: None,
-                ..Default::default()
-            })
-            .unwrap();
-
-        txid
-    }
-
-    #[test]
-    fn test_filter_available() {
-        let mut database = MemoryDatabase::new();
-        add_transaction(
-            &mut database,
-            vec![OutPoint::from_str(
-                "aad194c72fd5cfd16d23da9462930ca91e35df1cfee05242b62f4034f50c3d41:5",
-            )
-            .unwrap()],
-            vec![50_000],
-        );
-
-        let filtered =
-            filter_available(&database, database.iter_utxos().unwrap().into_iter()).unwrap();
-        assert_eq!(filtered, &[]);
-    }
-}

src/wallet/signer.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Generalized signers
 //!
@@ -30,12 +17,12 @@
 //! ```
 //! # use std::sync::Arc;
 //! # use std::str::FromStr;
+//! # use bitcoin::secp256k1::{Secp256k1, All};
 //! # use bitcoin::*;
 //! # use bitcoin::util::psbt;
-//! # use bitcoin::util::bip32::Fingerprint;
-//! # use magical::signer::*;
-//! # use magical::database::*;
-//! # use magical::*;
+//! # use bdk::signer::*;
+//! # use bdk::database::*;
+//! # use bdk::*;
 //! # #[derive(Debug)]
 //! # struct CustomHSM;
 //! # impl CustomHSM {
@@ -45,6 +32,9 @@
 //! #     fn connect() -> Self {
 //! #         CustomHSM
 //! #     }
+//! #     fn get_id(&self) -> SignerId {
+//! #         SignerId::Dummy(0)
+//! #     }
 //! # }
 //! #[derive(Debug)]
 //! struct CustomSigner {
@@ -62,6 +52,7 @@
 //!         &self,
 //!         psbt: &mut psbt::PartiallySignedTransaction,
 //!         input_index: Option<usize>,
+//!         _secp: &Secp256k1<All>,
 //!     ) -> Result<(), SignerError> {
 //!         let input_index = input_index.ok_or(SignerError::InputIndexOutOfRange)?;
 //!         self.device.sign_input(psbt, input_index)?;
@@ -69,6 +60,10 @@
 //!         Ok(())
 //!     }
 //!
+//!     fn id(&self, _secp: &Secp256k1<All>) -> SignerId {
+//!         self.device.get_id()
+//!     }
+//!
 //!     fn sign_whole_tx(&self) -> bool {
 //!         false
 //!     }
@@ -77,15 +72,14 @@
 //! let custom_signer = CustomSigner::connect();
 //!
 //! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)";
-//! let mut wallet: OfflineWallet<_> = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
+//! let mut wallet = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
 //! wallet.add_signer(
-//!     ScriptType::External,
-//!     Fingerprint::from_str("e30f11b8").unwrap().into(),
+//!     KeychainKind::External,
 //!     SignerOrdering(200),
-//!     Arc::new(Box::new(custom_signer))
+//!     Arc::new(custom_signer)
 //! );
 //!
-//! # Ok::<_, magical::Error>(())
+//! # Ok::<_, bdk::Error>(())
 //! ```
 
 use std::cmp::Ordering;
@@ -98,31 +92,36 @@ use bitcoin::blockdata::opcodes;
 use bitcoin::blockdata::script::Builder as ScriptBuilder;
 use bitcoin::hashes::{hash160, Hash};
 use bitcoin::secp256k1::{Message, Secp256k1};
-use bitcoin::util::bip32::{ExtendedPrivKey, Fingerprint};
+use bitcoin::util::bip32::{ChildNumber, DerivationPath, ExtendedPrivKey, Fingerprint};
 use bitcoin::util::{bip143, psbt};
-use bitcoin::{PrivateKey, SigHash, SigHashType};
+use bitcoin::{PrivateKey, Script, SigHash, SigHashType};
 
-use miniscript::descriptor::{DescriptorPublicKey, DescriptorSecretKey, DescriptorXKey, KeyMap};
+use miniscript::descriptor::{DescriptorSecretKey, DescriptorSinglePriv, DescriptorXKey, KeyMap};
 use miniscript::{Legacy, MiniscriptKey, Segwitv0};
 
+use super::utils::SecpCtx;
 use crate::descriptor::XKeyUtils;
 
 /// Identifier of a signer in the `SignersContainers`. Used as a key to find the right signer among
 /// multiple of them
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum SignerId<Pk: MiniscriptKey> {
-    PkHash(<Pk as MiniscriptKey>::Hash),
+#[derive(Debug, Clone, Ord, PartialOrd, PartialEq, Eq, Hash)]
+pub enum SignerId {
+    /// Bitcoin HASH160 (RIPEMD160 after SHA256) hash of an ECDSA public key
+    PkHash(hash160::Hash),
+    /// The fingerprint of a BIP32 extended key
     Fingerprint(Fingerprint),
+    /// Dummy identifier
+    Dummy(u64),
 }
 
-impl From<hash160::Hash> for SignerId<DescriptorPublicKey> {
-    fn from(hash: hash160::Hash) -> SignerId<DescriptorPublicKey> {
+impl From<hash160::Hash> for SignerId {
+    fn from(hash: hash160::Hash) -> SignerId {
         SignerId::PkHash(hash)
     }
 }
 
-impl From<Fingerprint> for SignerId<DescriptorPublicKey> {
-    fn from(fing: Fingerprint) -> SignerId<DescriptorPublicKey> {
+impl From<Fingerprint> for SignerId {
+    fn from(fing: Fingerprint) -> SignerId {
         SignerId::Fingerprint(fing)
     }
 }
@@ -132,10 +131,10 @@ impl From<Fingerprint> for SignerId<DescriptorPublicKey> {
 pub enum SignerError {
     /// The private key is missing for the required public key
     MissingKey,
+    /// The private key in use has the right fingerprint but derives differently than expected
+    InvalidKey,
     /// The user canceled the operation
     UserCanceled,
-    /// The sighash is missing in the PSBT input
-    MissingSighash,
     /// Input index is out of range
     InputIndexOutOfRange,
     /// The `non_witness_utxo` field of the transaction is required to sign this input
@@ -147,7 +146,13 @@ pub enum SignerError {
     /// The `witness_script` field of the transaction is requied to sign this input
     MissingWitnessScript,
     /// The fingerprint and derivation path are missing from the psbt input
-    MissingHDKeypath,
+    MissingHdKeypath,
+    /// The psbt contains a non-`SIGHASH_ALL` sighash in one of its input and the user hasn't
+    /// explicitly allowed them
+    ///
+    /// To enable signing transactions with non-standard sighashes set
+    /// [`SignOptions::allow_all_sighashes`] to `true`.
+    NonStandardSighash,
 }
 
 impl fmt::Display for SignerError {
@@ -162,7 +167,7 @@ impl std::error::Error for SignerError {}
 ///
 /// This trait can be implemented to provide customized signers to the wallet. For an example see
 /// [`this module`](crate::wallet::signer)'s documentation.
-pub trait Signer: fmt::Debug {
+pub trait Signer: fmt::Debug + Send + Sync {
     /// Sign a PSBT
     ///
     /// The `input_index` argument is only provided if the wallet doesn't declare to sign the whole
@@ -172,12 +177,19 @@ pub trait Signer: fmt::Debug {
         &self,
         psbt: &mut psbt::PartiallySignedTransaction,
         input_index: Option<usize>,
+        secp: &SecpCtx,
     ) -> Result<(), SignerError>;
 
     /// Return whether or not the signer signs the whole transaction in one go instead of every
     /// input individually
     fn sign_whole_tx(&self) -> bool;
 
+    /// Return the [`SignerId`] for this signer
+    ///
+    /// The [`SignerId`] can be used to lookup a signer in the [`Wallet`](crate::Wallet)'s signers map or to
+    /// compare two signers.
+    fn id(&self, secp: &SecpCtx) -> SignerId;
+
     /// Return the secret key for the signer
     ///
     /// This is used internally to reconstruct the original descriptor that may contain secrets.
@@ -193,32 +205,61 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
         &self,
         psbt: &mut psbt::PartiallySignedTransaction,
         input_index: Option<usize>,
+        secp: &SecpCtx,
     ) -> Result<(), SignerError> {
         let input_index = input_index.unwrap();
         if input_index >= psbt.inputs.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
-        let deriv_path = match psbt.inputs[input_index]
-            .hd_keypaths
+        if psbt.inputs[input_index].final_script_sig.is_some()
+            || psbt.inputs[input_index].final_script_witness.is_some()
+        {
+            return Ok(());
+        }
+
+        let (public_key, full_path) = match psbt.inputs[input_index]
+            .bip32_derivation
             .iter()
-            .filter_map(|(_, &(fingerprint, ref path))| self.matches(fingerprint.clone(), &path))
+            .filter_map(|(pk, &(fingerprint, ref path))| {
+                if self.matches(&(fingerprint, path.clone()), secp).is_some() {
+                    Some((pk, path))
+                } else {
+                    None
+                }
+            })
             .next()
         {
-            Some(deriv_path) => deriv_path,
-            None => return Ok(()), // TODO: should report an error maybe?
+            Some((pk, full_path)) => (pk, full_path.clone()),
+            None => return Ok(()),
         };
 
-        let ctx = Secp256k1::signing_only();
+        let derived_key = match self.origin.clone() {
+            Some((_fingerprint, origin_path)) => {
+                let deriv_path = DerivationPath::from(
+                    &full_path.into_iter().cloned().collect::<Vec<ChildNumber>>()
+                        [origin_path.len()..],
+                );
+                self.xkey.derive_priv(secp, &deriv_path).unwrap()
+            }
+            None => self.xkey.derive_priv(secp, &full_path).unwrap(),
+        };
 
-        let derived_key = self.xkey.derive_priv(&ctx, &deriv_path).unwrap();
-        derived_key.private_key.sign(psbt, Some(input_index))
+        if &derived_key.private_key.public_key(secp) != public_key {
+            Err(SignerError::InvalidKey)
+        } else {
+            derived_key.private_key.sign(psbt, Some(input_index), secp)
+        }
     }
 
     fn sign_whole_tx(&self) -> bool {
         false
     }
 
+    fn id(&self, secp: &SecpCtx) -> SignerId {
+        SignerId::from(self.root_fingerprint(secp))
+    }
+
     fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
         Some(DescriptorSecretKey::XPrv(self.clone()))
     }
@@ -229,15 +270,20 @@ impl Signer for PrivateKey {
         &self,
         psbt: &mut psbt::PartiallySignedTransaction,
         input_index: Option<usize>,
+        secp: &SecpCtx,
     ) -> Result<(), SignerError> {
         let input_index = input_index.unwrap();
-        if input_index >= psbt.inputs.len() {
+        if input_index >= psbt.inputs.len() || input_index >= psbt.global.unsigned_tx.input.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
-        let ctx = Secp256k1::signing_only();
+        if psbt.inputs[input_index].final_script_sig.is_some()
+            || psbt.inputs[input_index].final_script_witness.is_some()
+        {
+            return Ok(());
+        }
 
-        let pubkey = self.public_key(&ctx);
+        let pubkey = self.public_key(secp);
         if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
             return Ok(());
         }
@@ -251,7 +297,7 @@ impl Signer for PrivateKey {
             None => Legacy::sighash(psbt, input_index)?,
         };
 
-        let signature = ctx.sign(
+        let signature = secp.sign(
             &Message::from_slice(&hash.into_inner()[..]).unwrap(),
             &self.key,
         );
@@ -271,8 +317,15 @@ impl Signer for PrivateKey {
         false
     }
 
+    fn id(&self, secp: &SecpCtx) -> SignerId {
+        SignerId::from(self.public_key(secp).to_pubkeyhash())
+    }
+
     fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
-        Some(DescriptorSecretKey::PrivKey(self.clone()))
+        Some(DescriptorSecretKey::SinglePriv(DescriptorSinglePriv {
+            key: *self,
+            origin: None,
+        }))
     }
 }
 
@@ -291,13 +344,13 @@ impl std::default::Default for SignerOrdering {
 }
 
 #[derive(Debug, Clone)]
-struct SignersContainerKey<Pk: MiniscriptKey> {
-    id: SignerId<Pk>,
+struct SignersContainerKey {
+    id: SignerId,
     ordering: SignerOrdering,
 }
 
-impl<Pk: MiniscriptKey> From<(SignerId<Pk>, SignerOrdering)> for SignersContainerKey<Pk> {
-    fn from(tuple: (SignerId<Pk>, SignerOrdering)) -> Self {
+impl From<(SignerId, SignerOrdering)> for SignersContainerKey {
+    fn from(tuple: (SignerId, SignerOrdering)) -> Self {
         SignersContainerKey {
             id: tuple.0,
             ordering: tuple.1,
@@ -307,39 +360,35 @@ impl<Pk: MiniscriptKey> From<(SignerId<Pk>, SignerOrdering)> for SignersContaine
 
 /// Container for multiple signers
 #[derive(Debug, Default, Clone)]
-pub struct SignersContainer<Pk: MiniscriptKey>(
-    BTreeMap<SignersContainerKey<Pk>, Arc<Box<dyn Signer>>>,
-);
+pub struct SignersContainer(BTreeMap<SignersContainerKey, Arc<dyn Signer>>);
 
-impl SignersContainer<DescriptorPublicKey> {
-    pub fn as_key_map(&self) -> KeyMap {
+impl SignersContainer {
+    /// Create a map of public keys to secret keys
+    pub fn as_key_map(&self, secp: &SecpCtx) -> KeyMap {
         self.0
             .values()
             .filter_map(|signer| signer.descriptor_secret_key())
-            .filter_map(|secret| secret.as_public().ok().map(|public| (public, secret)))
+            .filter_map(|secret| secret.as_public(secp).ok().map(|public| (public, secret)))
             .collect()
     }
 }
 
-impl From<KeyMap> for SignersContainer<DescriptorPublicKey> {
-    fn from(keymap: KeyMap) -> SignersContainer<DescriptorPublicKey> {
+impl From<KeyMap> for SignersContainer {
+    fn from(keymap: KeyMap) -> SignersContainer {
+        let secp = Secp256k1::new();
         let mut container = SignersContainer::new();
 
         for (_, secret) in keymap {
             match secret {
-                DescriptorSecretKey::PrivKey(private_key) => container.add_external(
-                    SignerId::from(
-                        private_key
-                            .public_key(&Secp256k1::signing_only())
-                            .to_pubkeyhash(),
-                    ),
+                DescriptorSecretKey::SinglePriv(private_key) => container.add_external(
+                    SignerId::from(private_key.key.public_key(&secp).to_pubkeyhash()),
                     SignerOrdering::default(),
-                    Arc::new(Box::new(private_key)),
+                    Arc::new(private_key.key),
                 ),
                 DescriptorSecretKey::XPrv(xprv) => container.add_external(
-                    SignerId::from(xprv.root_fingerprint()),
+                    SignerId::from(xprv.root_fingerprint(&secp)),
                     SignerOrdering::default(),
-                    Arc::new(Box::new(xprv)),
+                    Arc::new(xprv),
                 ),
             };
         }
@@ -348,34 +397,30 @@ impl From<KeyMap> for SignersContainer<DescriptorPublicKey> {
     }
 }
 
-impl<Pk: MiniscriptKey> SignersContainer<Pk> {
+impl SignersContainer {
     /// Default constructor
     pub fn new() -> Self {
         SignersContainer(Default::default())
     }
 
     /// Adds an external signer to the container for the specified id. Optionally returns the
-    /// signer that was previosuly in the container, if any
+    /// signer that was previously in the container, if any
     pub fn add_external(
         &mut self,
-        id: SignerId<Pk>,
+        id: SignerId,
         ordering: SignerOrdering,
-        signer: Arc<Box<dyn Signer>>,
-    ) -> Option<Arc<Box<dyn Signer>>> {
+        signer: Arc<dyn Signer>,
+    ) -> Option<Arc<dyn Signer>> {
         self.0.insert((id, ordering).into(), signer)
     }
 
     /// Removes a signer from the container and returns it
-    pub fn remove(
-        &mut self,
-        id: SignerId<Pk>,
-        ordering: SignerOrdering,
-    ) -> Option<Arc<Box<dyn Signer>>> {
+    pub fn remove(&mut self, id: SignerId, ordering: SignerOrdering) -> Option<Arc<dyn Signer>> {
         self.0.remove(&(id, ordering).into())
     }
 
     /// Returns the list of identifiers of all the signers in the container
-    pub fn ids(&self) -> Vec<&SignerId<Pk>> {
+    pub fn ids(&self) -> Vec<&SignerId> {
         self.0
             .keys()
             .map(|SignersContainerKey { id, .. }| id)
@@ -383,19 +428,64 @@ impl<Pk: MiniscriptKey> SignersContainer<Pk> {
     }
 
     /// Returns the list of signers in the container, sorted by lowest to highest `ordering`
-    pub fn signers(&self) -> Vec<&Arc<Box<dyn Signer>>> {
+    pub fn signers(&self) -> Vec<&Arc<dyn Signer>> {
         self.0.values().collect()
     }
 
     /// Finds the signer with lowest ordering for a given id in the container.
-    pub fn find(&self, id: SignerId<Pk>) -> Option<&Arc<Box<dyn Signer>>> {
+    pub fn find(&self, id: SignerId) -> Option<&Arc<dyn Signer>> {
         self.0
             .range((
                 Included(&(id.clone(), SignerOrdering(0)).into()),
-                Included(&(id, SignerOrdering(usize::MAX)).into()),
+                Included(&(id.clone(), SignerOrdering(usize::MAX)).into()),
             ))
+            .filter(|(k, _)| k.id == id)
             .map(|(_, v)| v)
-            .nth(0)
+            .next()
+    }
+}
+
+/// Options for a software signer
+///
+/// Adjust the behavior of our software signers and the way a transaction is finalized
+#[derive(Debug, Clone)]
+pub struct SignOptions {
+    /// Whether the signer should trust the `witness_utxo`, if the `non_witness_utxo` hasn't been
+    /// provided
+    ///
+    /// Defaults to `false` to mitigate the "SegWit bug" which chould trick the wallet into
+    /// paying a fee larger than expected.
+    ///
+    /// Some wallets, especially if relatively old, might not provide the `non_witness_utxo` for
+    /// SegWit transactions in the PSBT they generate: in those cases setting this to `true`
+    /// should correctly produce a signature, at the expense of an increased trust in the creator
+    /// of the PSBT.
+    ///
+    /// For more details see: <https://blog.trezor.io/details-of-firmware-updates-for-trezor-one-version-1-9-1-and-trezor-model-t-version-2-3-1-1eba8f60f2dd>
+    pub trust_witness_utxo: bool,
+
+    /// Whether the wallet should assume a specific height has been reached when trying to finalize
+    /// a transaction
+    ///
+    /// The wallet will only "use" a timelock to satisfy the spending policy of an input if the
+    /// timelock height has already been reached. This option allows overriding the "current height" to let the
+    /// wallet use timelocks in the future to spend a coin.
+    pub assume_height: Option<u32>,
+
+    /// Whether the signer should use the `sighash_type` set in the PSBT when signing, no matter
+    /// what its value is
+    ///
+    /// Defaults to `false` which will only allow signing using `SIGHASH_ALL`.
+    pub allow_all_sighashes: bool,
+}
+
+impl Default for SignOptions {
+    fn default() -> Self {
+        SignOptions {
+            trust_witness_utxo: false,
+            assume_height: None,
+            allow_all_sighashes: false,
+        }
     }
 }
 
@@ -411,17 +501,17 @@ impl ComputeSighash for Legacy {
         psbt: &psbt::PartiallySignedTransaction,
         input_index: usize,
     ) -> Result<(SigHash, SigHashType), SignerError> {
-        if input_index >= psbt.inputs.len() {
+        if input_index >= psbt.inputs.len() || input_index >= psbt.global.unsigned_tx.input.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
         let psbt_input = &psbt.inputs[input_index];
         let tx_input = &psbt.global.unsigned_tx.input[input_index];
 
-        let sighash = psbt_input.sighash_type.ok_or(SignerError::MissingSighash)?;
-        let script = match &psbt_input.redeem_script {
-            &Some(ref redeem_script) => redeem_script.clone(),
-            &None => {
+        let sighash = psbt_input.sighash_type.unwrap_or(SigHashType::All);
+        let script = match psbt_input.redeem_script {
+            Some(ref redeem_script) => redeem_script.clone(),
+            None => {
                 let non_witness_utxo = psbt_input
                     .non_witness_utxo
                     .as_ref()
@@ -444,36 +534,64 @@ impl ComputeSighash for Legacy {
     }
 }
 
+fn p2wpkh_script_code(script: &Script) -> Script {
+    ScriptBuilder::new()
+        .push_opcode(opcodes::all::OP_DUP)
+        .push_opcode(opcodes::all::OP_HASH160)
+        .push_slice(&script[2..])
+        .push_opcode(opcodes::all::OP_EQUALVERIFY)
+        .push_opcode(opcodes::all::OP_CHECKSIG)
+        .into_script()
+}
+
 impl ComputeSighash for Segwitv0 {
     fn sighash(
         psbt: &psbt::PartiallySignedTransaction,
         input_index: usize,
     ) -> Result<(SigHash, SigHashType), SignerError> {
-        if input_index >= psbt.inputs.len() {
+        if input_index >= psbt.inputs.len() || input_index >= psbt.global.unsigned_tx.input.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
         let psbt_input = &psbt.inputs[input_index];
+        let tx_input = &psbt.global.unsigned_tx.input[input_index];
 
-        let sighash = psbt_input.sighash_type.ok_or(SignerError::MissingSighash)?;
+        let sighash = psbt_input.sighash_type.unwrap_or(SigHashType::All);
 
-        let witness_utxo = psbt_input
-            .witness_utxo
-            .as_ref()
-            .ok_or(SignerError::MissingNonWitnessUtxo)?;
-        let value = witness_utxo.value;
+        // Always try first with the non-witness utxo
+        let utxo = if let Some(prev_tx) = &psbt_input.non_witness_utxo {
+            // Check the provided prev-tx
+            if prev_tx.txid() != tx_input.previous_output.txid {
+                return Err(SignerError::InvalidNonWitnessUtxo);
+            }
 
-        let script = match &psbt_input.witness_script {
-            &Some(ref witness_script) => witness_script.clone(),
-            &None => {
-                if witness_utxo.script_pubkey.is_v0_p2wpkh() {
-                    ScriptBuilder::new()
-                        .push_opcode(opcodes::all::OP_DUP)
-                        .push_opcode(opcodes::all::OP_HASH160)
-                        .push_slice(&witness_utxo.script_pubkey[2..])
-                        .push_opcode(opcodes::all::OP_EQUALVERIFY)
-                        .push_opcode(opcodes::all::OP_CHECKSIG)
-                        .into_script()
+            // The output should be present, if it's missing the `non_witness_utxo` is invalid
+            prev_tx
+                .output
+                .get(tx_input.previous_output.vout as usize)
+                .ok_or(SignerError::InvalidNonWitnessUtxo)?
+        } else if let Some(witness_utxo) = &psbt_input.witness_utxo {
+            // Fallback to the witness_utxo. If we aren't allowed to use it, signing should fail
+            // before we get to this point
+            witness_utxo
+        } else {
+            // Nothing has been provided
+            return Err(SignerError::MissingNonWitnessUtxo);
+        };
+        let value = utxo.value;
+
+        let script = match psbt_input.witness_script {
+            Some(ref witness_script) => witness_script.clone(),
+            None => {
+                if utxo.script_pubkey.is_v0_p2wpkh() {
+                    p2wpkh_script_code(&utxo.script_pubkey)
+                } else if psbt_input
+                    .redeem_script
+                    .as_ref()
+                    .map(Script::is_v0_p2wpkh)
+                    .unwrap_or(false)
+                {
+                    p2wpkh_script_code(psbt_input.redeem_script.as_ref().unwrap())
                 } else {
                     return Err(SignerError::MissingWitnessScript);
                 }
@@ -492,22 +610,157 @@ impl ComputeSighash for Segwitv0 {
     }
 }
 
-impl<Pk: MiniscriptKey> PartialOrd for SignersContainerKey<Pk> {
+impl PartialOrd for SignersContainerKey {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<Pk: MiniscriptKey> Ord for SignersContainerKey<Pk> {
+impl Ord for SignersContainerKey {
     fn cmp(&self, other: &Self) -> Ordering {
-        self.ordering.cmp(&other.ordering)
+        self.ordering
+            .cmp(&other.ordering)
+            .then(self.id.cmp(&other.id))
     }
 }
 
-impl<Pk: MiniscriptKey> PartialEq for SignersContainerKey<Pk> {
+impl PartialEq for SignersContainerKey {
     fn eq(&self, other: &Self) -> bool {
-        self.ordering == other.ordering
+        self.id == other.id && self.ordering == other.ordering
     }
 }
 
-impl<Pk: MiniscriptKey> Eq for SignersContainerKey<Pk> {}
+impl Eq for SignersContainerKey {}
+
+#[cfg(test)]
+mod signers_container_tests {
+    use super::*;
+    use crate::descriptor;
+    use crate::descriptor::IntoWalletDescriptor;
+    use crate::keys::{DescriptorKey, IntoDescriptorKey};
+    use bitcoin::secp256k1::{All, Secp256k1};
+    use bitcoin::util::bip32;
+    use bitcoin::util::psbt::PartiallySignedTransaction;
+    use bitcoin::Network;
+    use miniscript::ScriptContext;
+    use std::str::FromStr;
+
+    fn is_equal(this: &Arc<dyn Signer>, that: &Arc<DummySigner>) -> bool {
+        let secp = Secp256k1::new();
+        this.id(&secp) == that.id(&secp)
+    }
+
+    // Signers added with the same ordering (like `Ordering::default`) created from `KeyMap`
+    // should be preserved and not overwritten.
+    // This happens usually when a set of signers is created from a descriptor with private keys.
+    #[test]
+    fn signers_with_same_ordering() {
+        let secp = Secp256k1::new();
+
+        let (prvkey1, _, _) = setup_keys(TPRV0_STR);
+        let (prvkey2, _, _) = setup_keys(TPRV1_STR);
+        let desc = descriptor!(sh(multi(2, prvkey1, prvkey2))).unwrap();
+        let (_, keymap) = desc
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .unwrap();
+
+        let signers = SignersContainer::from(keymap);
+        assert_eq!(signers.ids().len(), 2);
+
+        let signers = signers.signers();
+        assert_eq!(signers.len(), 2);
+    }
+
+    #[test]
+    fn signers_sorted_by_ordering() {
+        let mut signers = SignersContainer::new();
+        let signer1 = Arc::new(DummySigner { number: 1 });
+        let signer2 = Arc::new(DummySigner { number: 2 });
+        let signer3 = Arc::new(DummySigner { number: 3 });
+
+        // Mixed order insertions verifies we are not inserting at head or tail.
+        signers.add_external(SignerId::Dummy(2), SignerOrdering(2), signer2.clone());
+        signers.add_external(SignerId::Dummy(1), SignerOrdering(1), signer1.clone());
+        signers.add_external(SignerId::Dummy(3), SignerOrdering(3), signer3.clone());
+
+        // Check that signers are sorted from lowest to highest ordering
+        let signers = signers.signers();
+
+        assert!(is_equal(signers[0], &signer1));
+        assert!(is_equal(signers[1], &signer2));
+        assert!(is_equal(signers[2], &signer3));
+    }
+
+    #[test]
+    fn find_signer_by_id() {
+        let mut signers = SignersContainer::new();
+        let signer1 = Arc::new(DummySigner { number: 1 });
+        let signer2 = Arc::new(DummySigner { number: 2 });
+        let signer3 = Arc::new(DummySigner { number: 3 });
+        let signer4 = Arc::new(DummySigner { number: 3 }); // Same ID as `signer3` but will use lower ordering.
+
+        let id1 = SignerId::Dummy(1);
+        let id2 = SignerId::Dummy(2);
+        let id3 = SignerId::Dummy(3);
+        let id_nonexistent = SignerId::Dummy(999);
+
+        signers.add_external(id1.clone(), SignerOrdering(1), signer1.clone());
+        signers.add_external(id2.clone(), SignerOrdering(2), signer2.clone());
+        signers.add_external(id3.clone(), SignerOrdering(3), signer3.clone());
+
+        assert!(matches!(signers.find(id1), Some(signer) if is_equal(signer, &signer1)));
+        assert!(matches!(signers.find(id2), Some(signer) if is_equal(signer, &signer2)));
+        assert!(matches!(signers.find(id3.clone()), Some(signer) if is_equal(signer, &signer3)));
+
+        // The `signer4` has the same ID as `signer3` but lower ordering.
+        // It should be found by `id3` instead of `signer3`.
+        signers.add_external(id3.clone(), SignerOrdering(2), signer4.clone());
+        assert!(matches!(signers.find(id3), Some(signer) if is_equal(signer, &signer4)));
+
+        // Can't find anything with ID that doesn't exist
+        assert!(matches!(signers.find(id_nonexistent), None));
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    struct DummySigner {
+        number: u64,
+    }
+
+    impl Signer for DummySigner {
+        fn sign(
+            &self,
+            _psbt: &mut PartiallySignedTransaction,
+            _input_index: Option<usize>,
+            _secp: &SecpCtx,
+        ) -> Result<(), SignerError> {
+            Ok(())
+        }
+
+        fn id(&self, _secp: &SecpCtx) -> SignerId {
+            SignerId::Dummy(self.number)
+        }
+
+        fn sign_whole_tx(&self) -> bool {
+            true
+        }
+    }
+
+    const TPRV0_STR:&str = "tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf";
+    const TPRV1_STR:&str = "tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N";
+
+    const PATH: &str = "m/44'/1'/0'/0";
+
+    fn setup_keys<Ctx: ScriptContext>(
+        tprv: &str,
+    ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
+        let secp: Secp256k1<All> = Secp256k1::new();
+        let path = bip32::DerivationPath::from_str(PATH).unwrap();
+        let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
+        let tpub = bip32::ExtendedPubKey::from_private(&secp, &tprv);
+        let fingerprint = tprv.fingerprint(&secp);
+        let prvkey = (tprv, path.clone()).into_descriptor_key().unwrap();
+        let pubkey = (tpub, path).into_descriptor_key().unwrap();
+
+        (prvkey, pubkey, fingerprint)
+    }
+}

src/wallet/time.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Cross-platform time
 //!

src/wallet/tx_builder.rs

@@ -1,26 +1,13 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
 //! Transaction builder
 //!
@@ -29,165 +16,522 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use magical::*;
+//! # use bdk::*;
+//! # use bdk::wallet::tx_builder::CreateTx;
 //! # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
-//! // Create a transaction with one output to `to_address` of 50_000 satoshi, with a custom fee rate
-//! // of 5.0 satoshi/vbyte, only spending non-change outputs and with RBF signaling
-//! // enabled
-//! let builder = TxBuilder::with_recipients(vec![(to_address.script_pubkey(), 50_000)])
+//! # let wallet = doctest_wallet!();
+//! // create a TxBuilder from a wallet
+//! let mut tx_builder = wallet.build_tx();
+//!
+//! tx_builder
+//!     // Create a transaction with one output to `to_address` of 50_000 satoshi
+//!     .add_recipient(to_address.script_pubkey(), 50_000)
+//!     // With a custom fee rate of 5.0 satoshi/vbyte
 //!     .fee_rate(FeeRate::from_sat_per_vb(5.0))
+//!     // Only spend non-change outputs
 //!     .do_not_spend_change()
+//!     // Turn on RBF signaling
 //!     .enable_rbf();
+//! let (psbt, tx_details) = tx_builder.finish()?;
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::collections::BTreeMap;
+use std::collections::HashSet;
 use std::default::Default;
+use std::marker::PhantomData;
 
+use bitcoin::util::psbt::{self, PartiallySignedTransaction as Psbt};
 use bitcoin::{OutPoint, Script, SigHashType, Transaction};
 
+use miniscript::descriptor::DescriptorTrait;
+
 use super::coin_selection::{CoinSelectionAlgorithm, DefaultCoinSelectionAlgorithm};
-use crate::types::{FeeRate, UTXO};
+use crate::{database::BatchDatabase, Error, Utxo, Wallet};
+use crate::{
+    types::{FeeRate, KeychainKind, LocalUtxo, WeightedUtxo},
+    TransactionDetails,
+};
+/// Context in which the [`TxBuilder`] is valid
+pub trait TxBuilderContext: std::fmt::Debug + Default + Clone {}
+
+/// Marker type to indicate the [`TxBuilder`] is being used to create a new transaction (as opposed
+/// to bumping the fee of an existing one).
+#[derive(Debug, Default, Clone)]
+pub struct CreateTx;
+impl TxBuilderContext for CreateTx {}
+
+/// Marker type to indicate the [`TxBuilder`] is being used to bump the fee of an existing transaction.
+#[derive(Debug, Default, Clone)]
+pub struct BumpFee;
+impl TxBuilderContext for BumpFee {}
 
 /// A transaction builder
 ///
-/// This structure contains the configuration that the wallet must follow to build a transaction.
+/// A `TxBuilder` is created by calling [`build_tx`] or [`build_fee_bump`] on a wallet. After
+/// assigning it, you set options on it until finally calling [`finish`] to consume the builder and
+/// generate the transaction.
 ///
-/// For an example see [this module](super::tx_builder)'s documentation;
-#[derive(Debug, Default)]
-pub struct TxBuilder<Cs: CoinSelectionAlgorithm> {
+/// Each option setting method on `TxBuilder` takes and returns `&mut self` so you can chain calls
+/// as in the following example:
+///
+/// ```
+/// # use bdk::*;
+/// # use bdk::wallet::tx_builder::*;
+/// # use bitcoin::*;
+/// # use core::str::FromStr;
+/// # let wallet = doctest_wallet!();
+/// # let addr1 = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+/// # let addr2 = addr1.clone();
+/// // chaining
+/// let (psbt1, details) = {
+///     let mut builder = wallet.build_tx();
+///     builder
+///         .ordering(TxOrdering::Untouched)
+///         .add_recipient(addr1.script_pubkey(), 50_000)
+///         .add_recipient(addr2.script_pubkey(), 50_000);
+///     builder.finish()?
+/// };
+///
+/// // non-chaining
+/// let (psbt2, details) = {
+///     let mut builder = wallet.build_tx();
+///     builder.ordering(TxOrdering::Untouched);
+///     for addr in &[addr1, addr2] {
+///         builder.add_recipient(addr.script_pubkey(), 50_000);
+///     }
+///     builder.finish()?
+/// };
+///
+/// assert_eq!(
+///     psbt1.global.unsigned_tx.output[..2],
+///     psbt2.global.unsigned_tx.output[..2]
+/// );
+/// # Ok::<(), bdk::Error>(())
+/// ```
+///
+/// At the moment [`coin_selection`] is an exception to the rule as it consumes `self`.
+/// This means it is usually best to call [`coin_selection`] on the return value of `build_tx` before assigning it.
+///
+/// For further examples see [this module](super::tx_builder)'s documentation;
+///
+/// [`build_tx`]: Wallet::build_tx
+/// [`build_fee_bump`]: Wallet::build_fee_bump
+/// [`finish`]: Self::finish
+/// [`coin_selection`]: Self::coin_selection
+#[derive(Debug)]
+pub struct TxBuilder<'a, B, D, Cs, Ctx> {
+    pub(crate) wallet: &'a Wallet<B, D>,
+    pub(crate) params: TxParams,
+    pub(crate) coin_selection: Cs,
+    pub(crate) phantom: PhantomData<Ctx>,
+}
+
+/// The parameters for transaction creation sans coin selection algorithm.
+//TODO: TxParams should eventually be exposed publicly.
+#[derive(Default, Debug, Clone)]
+pub(crate) struct TxParams {
     pub(crate) recipients: Vec<(Script, u64)>,
-    pub(crate) send_all: bool,
-    pub(crate) fee_rate: Option<FeeRate>,
-    pub(crate) policy_path: Option<BTreeMap<String, Vec<usize>>>,
-    pub(crate) utxos: Option<Vec<OutPoint>>,
-    pub(crate) unspendable: Option<Vec<OutPoint>>,
+    pub(crate) drain_wallet: bool,
+    pub(crate) single_recipient: Option<Script>,
+    pub(crate) fee_policy: Option<FeePolicy>,
+    pub(crate) internal_policy_path: Option<BTreeMap<String, Vec<usize>>>,
+    pub(crate) external_policy_path: Option<BTreeMap<String, Vec<usize>>>,
+    pub(crate) utxos: Vec<WeightedUtxo>,
+    pub(crate) unspendable: HashSet<OutPoint>,
+    pub(crate) manually_selected_only: bool,
     pub(crate) sighash: Option<SigHashType>,
     pub(crate) ordering: TxOrdering,
     pub(crate) locktime: Option<u32>,
-    pub(crate) rbf: Option<u32>,
+    pub(crate) rbf: Option<RbfValue>,
     pub(crate) version: Option<Version>,
     pub(crate) change_policy: ChangeSpendPolicy,
-    pub(crate) force_non_witness_utxo: bool,
-    pub(crate) coin_selection: Cs,
+    pub(crate) only_witness_utxo: bool,
+    pub(crate) add_global_xpubs: bool,
+    pub(crate) include_output_redeem_witness_script: bool,
+    pub(crate) bumping_fee: Option<PreviousFee>,
 }
 
-impl TxBuilder<DefaultCoinSelectionAlgorithm> {
-    /// Create an empty builder
-    pub fn new() -> Self {
-        Self::default()
-    }
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct PreviousFee {
+    pub absolute: u64,
+    pub rate: f32,
+}
 
-    /// Create a builder starting from a list of recipients
-    pub fn with_recipients(recipients: Vec<(Script, u64)>) -> Self {
-        Self::default().set_recipients(recipients)
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum FeePolicy {
+    FeeRate(FeeRate),
+    FeeAmount(u64),
+}
+
+impl std::default::Default for FeePolicy {
+    fn default() -> Self {
+        FeePolicy::FeeRate(FeeRate::default_min_relay_fee())
     }
 }
 
-impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
-    /// Replace the recipients already added with a new list
-    pub fn set_recipients(mut self, recipients: Vec<(Script, u64)>) -> Self {
-        self.recipients = recipients;
-        self
-    }
-
-    /// Add a recipient to the internal list
-    pub fn add_recipient(mut self, script_pubkey: Script, amount: u64) -> Self {
-        self.recipients.push((script_pubkey, amount));
-        self
-    }
-
-    /// Send all the selected utxos to a single output
-    ///
-    /// Adding more than one recipients with this option enabled will result in an error.
-    ///
-    /// The value associated with the only recipient is irrelevant and will be replaced by the wallet.
-    pub fn send_all(mut self) -> Self {
-        self.send_all = true;
-        self
+impl<'a, Cs: Clone, Ctx, B, D> Clone for TxBuilder<'a, B, D, Cs, Ctx> {
+    fn clone(&self) -> Self {
+        TxBuilder {
+            wallet: self.wallet,
+            params: self.params.clone(),
+            coin_selection: self.coin_selection.clone(),
+            phantom: PhantomData,
+        }
     }
+}
 
+// methods supported by both contexts, for any CoinSelectionAlgorithm
+impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext>
+    TxBuilder<'a, B, D, Cs, Ctx>
+{
     /// Set a custom fee rate
-    pub fn fee_rate(mut self, fee_rate: FeeRate) -> Self {
-        self.fee_rate = Some(fee_rate);
+    pub fn fee_rate(&mut self, fee_rate: FeeRate) -> &mut Self {
+        self.params.fee_policy = Some(FeePolicy::FeeRate(fee_rate));
         self
     }
 
-    /// Set the policy path to use while creating the transaction
+    /// Set an absolute fee
+    pub fn fee_absolute(&mut self, fee_amount: u64) -> &mut Self {
+        self.params.fee_policy = Some(FeePolicy::FeeAmount(fee_amount));
+        self
+    }
+
+    /// Set the policy path to use while creating the transaction for a given keychain.
     ///
     /// This method accepts a map where the key is the policy node id (see
     /// [`Policy::id`](crate::descriptor::Policy::id)) and the value is the list of the indexes of
     /// the items that are intended to be satisfied from the policy node (see
     /// [`SatisfiableItem::Thresh::items`](crate::descriptor::policy::SatisfiableItem::Thresh::items)).
-    pub fn policy_path(mut self, policy_path: BTreeMap<String, Vec<usize>>) -> Self {
-        self.policy_path = Some(policy_path);
+    ///
+    /// ## Example
+    ///
+    /// An example of when the policy path is needed is the following descriptor:
+    /// `wsh(thresh(2,pk(A),sj:and_v(v:pk(B),n:older(6)),snj:and_v(v:pk(C),after(630000))))`,
+    /// derived from the miniscript policy `thresh(2,pk(A),and(pk(B),older(6)),and(pk(C),after(630000)))`.
+    /// It declares three descriptor fragments, and at the top level it uses `thresh()` to
+    /// ensure that at least two of them are satisfied. The individual fragments are:
+    ///
+    /// 1. `pk(A)`
+    /// 2. `and(pk(B),older(6))`
+    /// 3. `and(pk(C),after(630000))`
+    ///
+    /// When those conditions are combined in pairs, it's clear that the transaction needs to be created
+    /// differently depending on how the user intends to satisfy the policy afterwards:
+    ///
+    /// * If fragments `1` and `2` are used, the transaction will need to use a specific
+    ///   `n_sequence` in order to spend an `OP_CSV` branch.
+    /// * If fragments `1` and `3` are used, the transaction will need to use a specific `locktime`
+    ///   in order to spend an `OP_CLTV` branch.
+    /// * If fragments `2` and `3` are used, the transaction will need both.
+    ///
+    /// When the spending policy is represented as a tree (see
+    /// [`Wallet::policies`](super::Wallet::policies)), every node
+    /// is assigned a unique identifier that can be used in the policy path to specify which of
+    /// the node's children the user intends to satisfy: for instance, assuming the `thresh()`
+    /// root node of this example has an id of `aabbccdd`, the policy path map would look like:
+    ///
+    /// `{ "aabbccdd" => [0, 1] }`
+    ///
+    /// where the key is the node's id, and the value is a list of the children that should be
+    /// used, in no particular order.
+    ///
+    /// If a particularly complex descriptor has multiple ambiguous thresholds in its structure,
+    /// multiple entries can be added to the map, one for each node that requires an explicit path.
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// # use std::collections::BTreeMap;
+    /// # use bitcoin::*;
+    /// # use bdk::*;
+    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// # let wallet = doctest_wallet!();
+    /// let mut path = BTreeMap::new();
+    /// path.insert("aabbccdd".to_string(), vec![0, 1]);
+    ///
+    /// let builder = wallet
+    ///     .build_tx()
+    ///     .add_recipient(to_address.script_pubkey(), 50_000)
+    ///     .policy_path(path, KeychainKind::External);
+    ///
+    /// # Ok::<(), bdk::Error>(())
+    /// ```
+    pub fn policy_path(
+        &mut self,
+        policy_path: BTreeMap<String, Vec<usize>>,
+        keychain: KeychainKind,
+    ) -> &mut Self {
+        let to_update = match keychain {
+            KeychainKind::Internal => &mut self.params.internal_policy_path,
+            KeychainKind::External => &mut self.params.external_policy_path,
+        };
+
+        *to_update = Some(policy_path);
         self
     }
 
-    /// Replace the internal list of utxos that **must** be spent with a new list
+    /// Add the list of outpoints to the internal list of UTXOs that **must** be spent.
+    ///
+    /// If an error occurs while adding any of the UTXOs then none of them are added and the error is returned.
     ///
     /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
     /// the "utxos" and the "unspendable" list, it will be spent.
-    pub fn utxos(mut self, utxos: Vec<OutPoint>) -> Self {
-        self.utxos = Some(utxos);
-        self
+    pub fn add_utxos(&mut self, outpoints: &[OutPoint]) -> Result<&mut Self, Error> {
+        let utxos = outpoints
+            .iter()
+            .map(|outpoint| self.wallet.get_utxo(*outpoint)?.ok_or(Error::UnknownUtxo))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        for utxo in utxos {
+            let descriptor = self.wallet.get_descriptor_for_keychain(utxo.keychain);
+            let satisfaction_weight = descriptor.max_satisfaction_weight().unwrap();
+            self.params.utxos.push(WeightedUtxo {
+                satisfaction_weight,
+                utxo: Utxo::Local(utxo),
+            });
+        }
+
+        Ok(self)
     }
 
     /// Add a utxo to the internal list of utxos that **must** be spent
     ///
     /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
     /// the "utxos" and the "unspendable" list, it will be spent.
-    pub fn add_utxo(mut self, utxo: OutPoint) -> Self {
-        self.utxos.get_or_insert(vec![]).push(utxo);
+    pub fn add_utxo(&mut self, outpoint: OutPoint) -> Result<&mut Self, Error> {
+        self.add_utxos(&[outpoint])
+    }
+
+    /// Add a foreign UTXO i.e. a UTXO not owned by this wallet.
+    ///
+    /// At a minimum to add a foreign UTXO we need:
+    ///
+    /// 1. `outpoint`: To add it to the raw transaction.
+    /// 2. `psbt_input`: To know the value.
+    /// 3. `satisfaction_weight`: To know how much weight/vbytes the input will add to the transaction for fee calculation.
+    ///
+    /// There are several security concerns about adding foregin UTXOs that application
+    /// developers should consider. First, how do you know the value of the input is correct? If a
+    /// `non_witness_utxo` is provided in the `psbt_input` then this method implicitly verifies the
+    /// value by checking it against the transaction. If only a `witness_utxo` is provided then this
+    /// method doesn't verify the value but just takes it as a given -- it is up to you to check
+    /// that whoever sent you the `input_psbt` was not lying!
+    ///
+    /// Secondly, you must somehow provide `satisfaction_weight` of the input. Depending on your
+    /// application it may be important that this be known precisely. If not, a malicious
+    /// counterparty may fool you into putting in a value that is too low, giving the transaction a
+    /// lower than expected feerate. They could also fool you into putting a value that is too high
+    /// causing you to pay a fee that is too high. The party who is broadcasting the transaction can
+    /// of course check the real input weight matches the expected weight prior to broadcasting.
+    ///
+    /// To guarantee the `satisfaction_weight` is correct, you can require the party providing the
+    /// `psbt_input` provide a miniscript descriptor for the input so you can check it against the
+    /// `script_pubkey` and then ask it for the [`max_satisfaction_weight`].
+    ///
+    /// This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+    ///
+    /// # Errors
+    ///
+    /// This method returns errors in the following circumstances:
+    ///
+    /// 1. The `psbt_input` does not contain a `witness_utxo` or `non_witness_utxo`.
+    /// 2. The data in `non_witness_utxo` does not match what is in `outpoint`.
+    ///
+    /// Note unless you set [`only_witness_utxo`] any `psbt_input` you pass to this method must
+    /// have `non_witness_utxo` set otherwise you will get an error when [`finish`] is called.
+    ///
+    /// [`only_witness_utxo`]: Self::only_witness_utxo
+    /// [`finish`]: Self::finish
+    /// [`max_satisfaction_weight`]: miniscript::Descriptor::max_satisfaction_weight
+    pub fn add_foreign_utxo(
+        &mut self,
+        outpoint: OutPoint,
+        psbt_input: psbt::Input,
+        satisfaction_weight: usize,
+    ) -> Result<&mut Self, Error> {
+        if psbt_input.witness_utxo.is_none() {
+            match psbt_input.non_witness_utxo.as_ref() {
+                Some(tx) => {
+                    if tx.txid() != outpoint.txid {
+                        return Err(Error::Generic(
+                            "Foreign utxo outpoint does not match PSBT input".into(),
+                        ));
+                    }
+                    if tx.output.len() <= outpoint.vout as usize {
+                        return Err(Error::InvalidOutpoint(outpoint));
+                    }
+                }
+                None => {
+                    return Err(Error::Generic(
+                        "Foreign utxo missing witness_utxo or non_witness_utxo".into(),
+                    ))
+                }
+            }
+        }
+
+        self.params.utxos.push(WeightedUtxo {
+            satisfaction_weight,
+            utxo: Utxo::Foreign {
+                outpoint,
+                psbt_input: Box::new(psbt_input),
+            },
+        });
+
+        Ok(self)
+    }
+
+    /// Only spend utxos added by [`add_utxo`].
+    ///
+    /// The wallet will **not** add additional utxos to the transaction even if they are needed to
+    /// make the transaction valid.
+    ///
+    /// [`add_utxo`]: Self::add_utxo
+    pub fn manually_selected_only(&mut self) -> &mut Self {
+        self.params.manually_selected_only = true;
         self
     }
 
     /// Replace the internal list of unspendable utxos with a new list
     ///
-    /// It's important to note that the "must-be-spent" utxos added with [`TxBuilder::utxos`] and
-    /// [`TxBuilder::add_utxo`] have priority over these. See the docs of the two linked methods
-    /// for more details.
-    pub fn unspendable(mut self, unspendable: Vec<OutPoint>) -> Self {
-        self.unspendable = Some(unspendable);
+    /// It's important to note that the "must-be-spent" utxos added with [`TxBuilder::add_utxo`]
+    /// have priority over these. See the docs of the two linked methods for more details.
+    pub fn unspendable(&mut self, unspendable: Vec<OutPoint>) -> &mut Self {
+        self.params.unspendable = unspendable.into_iter().collect();
         self
     }
 
     /// Add a utxo to the internal list of unspendable utxos
     ///
-    /// It's important to note that the "must-be-spent" utxos added with [`TxBuilder::utxos`] and
-    /// [`TxBuilder::add_utxo`] have priority over this. See the docs of the two linked methods
-    /// for more details.
-    pub fn add_unspendable(mut self, unspendable: OutPoint) -> Self {
-        self.unspendable.get_or_insert(vec![]).push(unspendable);
+    /// It's important to note that the "must-be-spent" utxos added with [`TxBuilder::add_utxo`]
+    /// have priority over this. See the docs of the two linked methods for more details.
+    pub fn add_unspendable(&mut self, unspendable: OutPoint) -> &mut Self {
+        self.params.unspendable.insert(unspendable);
         self
     }
 
     /// Sign with a specific sig hash
     ///
     /// **Use this option very carefully**
-    pub fn sighash(mut self, sighash: SigHashType) -> Self {
-        self.sighash = Some(sighash);
+    pub fn sighash(&mut self, sighash: SigHashType) -> &mut Self {
+        self.params.sighash = Some(sighash);
         self
     }
 
     /// Choose the ordering for inputs and outputs of the transaction
-    pub fn ordering(mut self, ordering: TxOrdering) -> Self {
-        self.ordering = ordering;
+    pub fn ordering(&mut self, ordering: TxOrdering) -> &mut Self {
+        self.params.ordering = ordering;
         self
     }
 
     /// Use a specific nLockTime while creating the transaction
     ///
     /// This can cause conflicts if the wallet's descriptors contain an "after" (OP_CLTV) operator.
-    pub fn nlocktime(mut self, locktime: u32) -> Self {
-        self.locktime = Some(locktime);
+    pub fn nlocktime(&mut self, locktime: u32) -> &mut Self {
+        self.params.locktime = Some(locktime);
         self
     }
 
+    /// Build a transaction with a specific version
+    ///
+    /// The `version` should always be greater than `0` and greater than `1` if the wallet's
+    /// descriptors contain an "older" (OP_CSV) operator.
+    pub fn version(&mut self, version: i32) -> &mut Self {
+        self.params.version = Some(Version(version));
+        self
+    }
+
+    /// Do not spend change outputs
+    ///
+    /// This effectively adds all the change outputs to the "unspendable" list. See
+    /// [`TxBuilder::unspendable`].
+    pub fn do_not_spend_change(&mut self) -> &mut Self {
+        self.params.change_policy = ChangeSpendPolicy::ChangeForbidden;
+        self
+    }
+
+    /// Only spend change outputs
+    ///
+    /// This effectively adds all the non-change outputs to the "unspendable" list. See
+    /// [`TxBuilder::unspendable`].
+    pub fn only_spend_change(&mut self) -> &mut Self {
+        self.params.change_policy = ChangeSpendPolicy::OnlyChange;
+        self
+    }
+
+    /// Set a specific [`ChangeSpendPolicy`]. See [`TxBuilder::do_not_spend_change`] and
+    /// [`TxBuilder::only_spend_change`] for some shortcuts.
+    pub fn change_policy(&mut self, change_policy: ChangeSpendPolicy) -> &mut Self {
+        self.params.change_policy = change_policy;
+        self
+    }
+
+    /// Only Fill-in the [`psbt::Input::witness_utxo`](bitcoin::util::psbt::Input::witness_utxo) field when spending from
+    /// SegWit descriptors.
+    ///
+    /// This reduces the size of the PSBT, but some signers might reject them due to the lack of
+    /// the `non_witness_utxo`.
+    pub fn only_witness_utxo(&mut self) -> &mut Self {
+        self.params.only_witness_utxo = true;
+        self
+    }
+
+    /// Fill-in the [`psbt::Output::redeem_script`](bitcoin::util::psbt::Output::redeem_script) and
+    /// [`psbt::Output::witness_script`](bitcoin::util::psbt::Output::witness_script) fields.
+    ///
+    /// This is useful for signers which always require it, like ColdCard hardware wallets.
+    pub fn include_output_redeem_witness_script(&mut self) -> &mut Self {
+        self.params.include_output_redeem_witness_script = true;
+        self
+    }
+
+    /// Fill-in the `PSBT_GLOBAL_XPUB` field with the extended keys contained in both the external
+    /// and internal descriptors
+    ///
+    /// This is useful for offline signers that take part to a multisig. Some hardware wallets like
+    /// BitBox and ColdCard are known to require this.
+    pub fn add_global_xpubs(&mut self) -> &mut Self {
+        self.params.add_global_xpubs = true;
+        self
+    }
+
+    /// Spend all the available inputs. This respects filters like [`TxBuilder::unspendable`] and the change policy.
+    pub fn drain_wallet(&mut self) -> &mut Self {
+        self.params.drain_wallet = true;
+        self
+    }
+
+    /// Choose the coin selection algorithm
+    ///
+    /// Overrides the [`DefaultCoinSelectionAlgorithm`](super::coin_selection::DefaultCoinSelectionAlgorithm).
+    ///
+    /// Note that this function consumes the builder and returns it so it is usually best to put this as the first call on the builder.
+    pub fn coin_selection<P: CoinSelectionAlgorithm<D>>(
+        self,
+        coin_selection: P,
+    ) -> TxBuilder<'a, B, D, P, Ctx> {
+        TxBuilder {
+            wallet: self.wallet,
+            params: self.params,
+            coin_selection,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Finish the building the transaction.
+    ///
+    /// Returns the [`BIP174`] "PSBT" and summary details about the transaction.
+    ///
+    /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
+    pub fn finish(self) -> Result<(Psbt, TransactionDetails), Error> {
+        self.wallet.create_tx(self.coin_selection, self.params)
+    }
+
     /// Enable signaling RBF
     ///
     /// This will use the default nSequence value of `0xFFFFFFFD`.
-    pub fn enable_rbf(self) -> Self {
-        self.enable_rbf_with_sequence(0xFFFFFFFD)
+    pub fn enable_rbf(&mut self) -> &mut Self {
+        self.params.rbf = Some(RbfValue::Default);
+        self
     }
 
     /// Enable signaling RBF with a specific nSequence value
@@ -197,74 +541,68 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
     ///
     /// If the `nsequence` is higher than `0xFFFFFFFD` an error will be thrown, since it would not
     /// be a valid nSequence to signal RBF.
-    pub fn enable_rbf_with_sequence(mut self, nsequence: u32) -> Self {
-        self.rbf = Some(nsequence);
+    pub fn enable_rbf_with_sequence(&mut self, nsequence: u32) -> &mut Self {
+        self.params.rbf = Some(RbfValue::Value(nsequence));
+        self
+    }
+}
+
+impl<'a, B, D: BatchDatabase, Cs: CoinSelectionAlgorithm<D>> TxBuilder<'a, B, D, Cs, CreateTx> {
+    /// Replace the recipients already added with a new list
+    pub fn set_recipients(&mut self, recipients: Vec<(Script, u64)>) -> &mut Self {
+        self.params.recipients = recipients;
         self
     }
 
-    /// Build a transaction with a specific version
+    /// Add a recipient to the internal list
+    pub fn add_recipient(&mut self, script_pubkey: Script, amount: u64) -> &mut Self {
+        self.params.recipients.push((script_pubkey, amount));
+        self
+    }
+
+    /// Set a single recipient that will get all the selected funds minus the fee. No change will
+    /// be created
     ///
-    /// The `version` should always be greater than `0` and greater than `1` if the wallet's
-    /// descriptors contain an "older" (OP_CSV) operator.
-    pub fn version(mut self, version: u32) -> Self {
-        self.version = Some(Version(version));
-        self
-    }
-
-    /// Do not spend change outputs
+    /// This method overrides any recipient set with [`set_recipients`](Self::set_recipients) or
+    /// [`add_recipient`](Self::add_recipient).
     ///
-    /// This effectively adds all the change outputs to the "unspendable" list. See
-    /// [`TxBuilder::unspendable`].
-    pub fn do_not_spend_change(mut self) -> Self {
-        self.change_policy = ChangeSpendPolicy::ChangeForbidden;
-        self
-    }
-
-    /// Only spend change outputs
+    /// It can only be used in conjunction with [`drain_wallet`](Self::drain_wallet) to send the
+    /// entire content of the wallet (minus filters) to a single recipient or with a
+    /// list of manually selected UTXOs by enabling [`manually_selected_only`](Self::manually_selected_only)
+    /// and selecting them with or [`add_utxo`](Self::add_utxo).
     ///
-    /// This effectively adds all the non-change outputs to the "unspendable" list. See
-    /// [`TxBuilder::unspendable`].
-    pub fn only_spend_change(mut self) -> Self {
-        self.change_policy = ChangeSpendPolicy::OnlyChange;
+    /// When bumping the fees of a transaction made with this option, the user should remeber to
+    /// add [`maintain_single_recipient`](Self::maintain_single_recipient) to correctly update the
+    /// single output instead of adding one more for the change.
+    pub fn set_single_recipient(&mut self, recipient: Script) -> &mut Self {
+        self.params.single_recipient = Some(recipient);
+        self.params.recipients.clear();
+
         self
     }
+}
 
-    /// Set a specific [`ChangeSpendPolicy`]. See [`TxBuilder::do_not_spend_change`] and
-    /// [`TxBuilder::only_spend_change`] for some shortcuts.
-    pub fn change_policy(mut self, change_policy: ChangeSpendPolicy) -> Self {
-        self.change_policy = change_policy;
-        self
-    }
-
-    /// Fill-in the [`psbt::Input::non_witness_utxo`](bitcoin::util::psbt::Input::non_witness_utxo) field even if the wallet only has SegWit
-    /// descriptors.
+// methods supported only by bump_fee
+impl<'a, B, D: BatchDatabase> TxBuilder<'a, B, D, DefaultCoinSelectionAlgorithm, BumpFee> {
+    /// Bump the fees of a transaction made with [`set_single_recipient`](Self::set_single_recipient)
     ///
-    /// This is useful for signers which always require it, like Trezor hardware wallets.
-    pub fn force_non_witness_utxo(mut self) -> Self {
-        self.force_non_witness_utxo = true;
-        self
-    }
-
-    /// Choose the coin selection algorithm
+    /// Unless extra inputs are specified with [`add_utxo`], this flag will make
+    /// `bump_fee` reduce the value of the existing output, or fail if it would be consumed
+    /// entirely given the higher new fee rate.
     ///
-    /// Overrides the [`DefaultCoinSelectionAlgorithm`](super::coin_selection::DefaultCoinSelectionAlgorithm).
-    pub fn coin_selection<P: CoinSelectionAlgorithm>(self, coin_selection: P) -> TxBuilder<P> {
-        TxBuilder {
-            recipients: self.recipients,
-            send_all: self.send_all,
-            fee_rate: self.fee_rate,
-            policy_path: self.policy_path,
-            utxos: self.utxos,
-            unspendable: self.unspendable,
-            sighash: self.sighash,
-            ordering: self.ordering,
-            locktime: self.locktime,
-            rbf: self.rbf,
-            version: self.version,
-            change_policy: self.change_policy,
-            force_non_witness_utxo: self.force_non_witness_utxo,
-            coin_selection,
+    /// If extra inputs are added and they are not entirely consumed in fees, a change output will not
+    /// be added; the existing output will simply grow in value.
+    ///
+    /// Fails if the transaction has more than one outputs.
+    ///
+    /// [`add_utxo`]: Self::add_utxo
+    pub fn maintain_single_recipient(&mut self) -> Result<&mut Self, Error> {
+        let mut recipients = self.params.recipients.drain(..).collect::<Vec<_>>();
+        if recipients.len() != 1 {
+            return Err(Error::SingleRecipientMultipleOutputs);
         }
+        self.params.single_recipient = Some(recipients.pop().unwrap().0);
+        Ok(self)
     }
 }
 
@@ -276,7 +614,7 @@ pub enum TxOrdering {
     /// Unchanged
     Untouched,
     /// BIP69 / Lexicographic
-    BIP69Lexicographic,
+    Bip69Lexicographic,
 }
 
 impl Default for TxOrdering {
@@ -286,6 +624,7 @@ impl Default for TxOrdering {
 }
 
 impl TxOrdering {
+    /// Sort transaction inputs and outputs by [`TxOrdering`] variant
     pub fn sort_tx(&self, tx: &mut Transaction) {
         match self {
             TxOrdering::Untouched => {}
@@ -301,7 +640,7 @@ impl TxOrdering {
 
                 tx.output.shuffle(&mut rng);
             }
-            TxOrdering::BIP69Lexicographic => {
+            TxOrdering::Bip69Lexicographic => {
                 tx.input.sort_unstable_by_key(|txin| {
                     (txin.previous_output.txid, txin.previous_output.vout)
                 });
@@ -316,7 +655,7 @@ impl TxOrdering {
 ///
 /// Has a default value of `1`
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
-pub(crate) struct Version(pub(crate) u32);
+pub(crate) struct Version(pub(crate) i32);
 
 impl Default for Version {
     fn default() -> Self {
@@ -324,6 +663,24 @@ impl Default for Version {
     }
 }
 
+/// RBF nSequence value
+///
+/// Has a default value of `0xFFFFFFFD`
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+pub(crate) enum RbfValue {
+    Default,
+    Value(u32),
+}
+
+impl RbfValue {
+    pub(crate) fn get_value(&self) -> u32 {
+        match self {
+            RbfValue::Default => 0xFFFFFFFD,
+            RbfValue::Value(v) => *v,
+        }
+    }
+}
+
 /// Policy regarding the use of change outputs when creating a transaction
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum ChangeSpendPolicy {
@@ -342,23 +699,23 @@ impl Default for ChangeSpendPolicy {
 }
 
 impl ChangeSpendPolicy {
-    pub(crate) fn filter_utxos<I: Iterator<Item = UTXO>>(&self, iter: I) -> Vec<UTXO> {
+    pub(crate) fn is_satisfied_by(&self, utxo: &LocalUtxo) -> bool {
         match self {
-            ChangeSpendPolicy::ChangeAllowed => iter.collect(),
-            ChangeSpendPolicy::OnlyChange => iter.filter(|utxo| utxo.is_internal).collect(),
-            ChangeSpendPolicy::ChangeForbidden => iter.filter(|utxo| !utxo.is_internal).collect(),
+            ChangeSpendPolicy::ChangeAllowed => true,
+            ChangeSpendPolicy::OnlyChange => utxo.keychain == KeychainKind::Internal,
+            ChangeSpendPolicy::ChangeForbidden => utxo.keychain == KeychainKind::External,
         }
     }
 }
 
 #[cfg(test)]
 mod test {
-    const ORDERING_TEST_TX: &'static str = "0200000003c26f3eb7932f7acddc5ddd26602b77e7516079b03090a16e2c2f54\
-                                            85d1fd600f0100000000ffffffffc26f3eb7932f7acddc5ddd26602b77e75160\
-                                            79b03090a16e2c2f5485d1fd600f0000000000ffffffff571fb3e02278217852\
-                                            dd5d299947e2b7354a639adc32ec1fa7b82cfb5dec530e0500000000ffffffff\
-                                            03e80300000000000002aaeee80300000000000001aa200300000000000001ff\
-                                            00000000";
+    const ORDERING_TEST_TX: &str = "0200000003c26f3eb7932f7acddc5ddd26602b77e7516079b03090a16e2c2f54\
+                                    85d1fd600f0100000000ffffffffc26f3eb7932f7acddc5ddd26602b77e75160\
+                                    79b03090a16e2c2f5485d1fd600f0000000000ffffffff571fb3e02278217852\
+                                    dd5d299947e2b7354a639adc32ec1fa7b82cfb5dec530e0500000000ffffffff\
+                                    03e80300000000000002aaeee80300000000000001aa200300000000000001ff\
+                                    00000000";
     macro_rules! ordering_test_tx {
         () => {
             deserialize::<bitcoin::Transaction>(&Vec::<u8>::from_hex(ORDERING_TEST_TX).unwrap())
@@ -402,9 +759,9 @@ mod test {
         use std::str::FromStr;
 
         let original_tx = ordering_test_tx!();
-        let mut tx = original_tx.clone();
+        let mut tx = original_tx;
 
-        TxOrdering::BIP69Lexicographic.sort_tx(&mut tx);
+        TxOrdering::Bip69Lexicographic.sort_tx(&mut tx);
 
         assert_eq!(
             tx.input[0].previous_output,
@@ -433,23 +790,23 @@ mod test {
         assert_eq!(tx.output[2].script_pubkey, From::from(vec![0xAA, 0xEE]));
     }
 
-    fn get_test_utxos() -> Vec<UTXO> {
+    fn get_test_utxos() -> Vec<LocalUtxo> {
         vec![
-            UTXO {
+            LocalUtxo {
                 outpoint: OutPoint {
                     txid: Default::default(),
                     vout: 0,
                 },
                 txout: Default::default(),
-                is_internal: false,
+                keychain: KeychainKind::External,
             },
-            UTXO {
+            LocalUtxo {
                 outpoint: OutPoint {
                     txid: Default::default(),
                     vout: 1,
                 },
                 txout: Default::default(),
-                is_internal: true,
+                keychain: KeychainKind::Internal,
             },
         ]
     }
@@ -457,27 +814,36 @@ mod test {
     #[test]
     fn test_change_spend_policy_default() {
         let change_spend_policy = ChangeSpendPolicy::default();
-        let filtered = change_spend_policy.filter_utxos(get_test_utxos().into_iter());
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .count();
 
-        assert_eq!(filtered.len(), 2);
+        assert_eq!(filtered, 2);
     }
 
     #[test]
     fn test_change_spend_policy_no_internal() {
         let change_spend_policy = ChangeSpendPolicy::ChangeForbidden;
-        let filtered = change_spend_policy.filter_utxos(get_test_utxos().into_iter());
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
 
         assert_eq!(filtered.len(), 1);
-        assert_eq!(filtered[0].is_internal, false);
+        assert_eq!(filtered[0].keychain, KeychainKind::External);
     }
 
     #[test]
     fn test_change_spend_policy_only_internal() {
         let change_spend_policy = ChangeSpendPolicy::OnlyChange;
-        let filtered = change_spend_policy.filter_utxos(get_test_utxos().into_iter());
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
 
         assert_eq!(filtered.len(), 1);
-        assert_eq!(filtered[0].is_internal, true);
+        assert_eq!(filtered[0].keychain, KeychainKind::Internal);
     }
 
     #[test]

src/wallet/utils.rs

@@ -1,31 +1,32 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
 //
-// Copyright (c) 2020 Magical Bitcoin
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
 
-use miniscript::{MiniscriptKey, Satisfier};
+use bitcoin::secp256k1::{All, Secp256k1};
+
+use miniscript::{MiniscriptKey, Satisfier, ToPublicKey};
 
 // De-facto standard "dust limit" (even though it should change based on the output type)
-const DUST_LIMIT_SATOSHI: u64 = 546;
+pub const DUST_LIMIT_SATOSHI: u64 = 546;
+
+// MSB of the nSequence. If set there's no consensus-constraint, so it must be disabled when
+// spending using CSV in order to enforce CSV rules
+pub(crate) const SEQUENCE_LOCKTIME_DISABLE_FLAG: u32 = 1 << 31;
+// When nSequence is lower than this flag the timelock is interpreted as block-height-based,
+// otherwise it's time-based
+pub(crate) const SEQUENCE_LOCKTIME_TYPE_FLAG: u32 = 1 << 22;
+// Mask for the bits used to express the timelock
+pub(crate) const SEQUENCE_LOCKTIME_MASK: u32 = 0x0000FFFF;
+
+// Threshold for nLockTime to be considered a block-height-based timelock rather than time-based
+pub(crate) const BLOCKS_TIMELOCK_THRESHOLD: u32 = 500000000;
 
 /// Trait to check if a value is below the dust limit
 // we implement this trait to make sure we don't mess up the comparison with off-by-one like a <
@@ -56,7 +57,45 @@ impl After {
     }
 }
 
-impl<Pk: MiniscriptKey> Satisfier<Pk> for After {
+pub(crate) fn check_nsequence_rbf(rbf: u32, csv: u32) -> bool {
+    // This flag cannot be set in the nSequence when spending using OP_CSV
+    if rbf & SEQUENCE_LOCKTIME_DISABLE_FLAG != 0 {
+        return false;
+    }
+
+    let mask = SEQUENCE_LOCKTIME_TYPE_FLAG | SEQUENCE_LOCKTIME_MASK;
+    let rbf = rbf & mask;
+    let csv = csv & mask;
+
+    // Both values should be represented in the same unit (either time-based or
+    // block-height based)
+    if (rbf < SEQUENCE_LOCKTIME_TYPE_FLAG) != (csv < SEQUENCE_LOCKTIME_TYPE_FLAG) {
+        return false;
+    }
+
+    // The value should be at least `csv`
+    if rbf < csv {
+        return false;
+    }
+
+    true
+}
+
+pub(crate) fn check_nlocktime(nlocktime: u32, required: u32) -> bool {
+    // Both values should be expressed in the same unit
+    if (nlocktime < BLOCKS_TIMELOCK_THRESHOLD) != (required < BLOCKS_TIMELOCK_THRESHOLD) {
+        return false;
+    }
+
+    // The value should be at least `required`
+    if nlocktime < required {
+        return false;
+    }
+
+    true
+}
+
+impl<Pk: MiniscriptKey + ToPublicKey> Satisfier<Pk> for After {
     fn check_after(&self, n: u32) -> bool {
         if let Some(current_height) = self.current_height {
             current_height >= n
@@ -86,7 +125,7 @@ impl Older {
     }
 }
 
-impl<Pk: MiniscriptKey> Satisfier<Pk> for Older {
+impl<Pk: MiniscriptKey + ToPublicKey> Satisfier<Pk> for Older {
     fn check_older(&self, n: u32) -> bool {
         if let Some(current_height) = self.current_height {
             // TODO: test >= / >
@@ -97,11 +136,14 @@ impl<Pk: MiniscriptKey> Satisfier<Pk> for Older {
     }
 }
 
+pub(crate) type SecpCtx = Secp256k1<All>;
+
 pub struct ChunksIterator<I: Iterator> {
     iter: I,
     size: usize,
 }
 
+#[cfg(any(feature = "electrum", feature = "esplora"))]
 impl<I: Iterator> ChunksIterator<I> {
     pub fn new(iter: I, size: usize) -> Self {
         ChunksIterator { iter, size }
@@ -132,6 +174,10 @@ impl<I: Iterator> Iterator for ChunksIterator<I> {
 
 #[cfg(test)]
 mod test {
+    use super::{
+        check_nlocktime, check_nsequence_rbf, BLOCKS_TIMELOCK_THRESHOLD,
+        SEQUENCE_LOCKTIME_TYPE_FLAG,
+    };
     use crate::types::FeeRate;
 
     #[test]
@@ -151,4 +197,70 @@ mod test {
         let fee = FeeRate::default_min_relay_fee();
         assert!((fee.as_sat_vb() - 1.0).abs() < 0.0001);
     }
+
+    #[test]
+    fn test_check_nsequence_rbf_msb_set() {
+        let result = check_nsequence_rbf(0x80000000, 5000);
+        assert!(!result);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_lt_csv() {
+        let result = check_nsequence_rbf(4000, 5000);
+        assert!(!result);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_different_unit() {
+        let result = check_nsequence_rbf(SEQUENCE_LOCKTIME_TYPE_FLAG + 5000, 5000);
+        assert!(!result);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_mask() {
+        let result = check_nsequence_rbf(0x3f + 10_000, 5000);
+        assert!(result);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_same_unit_blocks() {
+        let result = check_nsequence_rbf(10_000, 5000);
+        assert!(result);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_same_unit_time() {
+        let result = check_nsequence_rbf(
+            SEQUENCE_LOCKTIME_TYPE_FLAG + 10_000,
+            SEQUENCE_LOCKTIME_TYPE_FLAG + 5000,
+        );
+        assert!(result);
+    }
+
+    #[test]
+    fn test_check_nlocktime_lt_cltv() {
+        let result = check_nlocktime(4000, 5000);
+        assert!(!result);
+    }
+
+    #[test]
+    fn test_check_nlocktime_different_unit() {
+        let result = check_nlocktime(BLOCKS_TIMELOCK_THRESHOLD + 5000, 5000);
+        assert!(!result);
+    }
+
+    #[test]
+    fn test_check_nlocktime_same_unit_blocks() {
+        let result = check_nlocktime(10_000, 5000);
+        assert!(result);
+    }
+
+    #[test]
+    fn test_check_nlocktime_same_unit_time() {
+        let result = check_nlocktime(
+            BLOCKS_TIMELOCK_THRESHOLD + 10_000,
+            BLOCKS_TIMELOCK_THRESHOLD + 5000,
+        );
+        assert!(result);
+    }
 }

src/wallet/verify.rs

@@ -0,0 +1,186 @@
+// Bitcoin Dev Kit
+// Written in 2021 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Verify transactions against the consensus rules
+
+use std::collections::HashMap;
+use std::fmt;
+
+use bitcoin::consensus::serialize;
+use bitcoin::{OutPoint, Transaction, Txid};
+
+use crate::blockchain::Blockchain;
+use crate::database::Database;
+use crate::error::Error;
+
+/// Verify a transaction against the consensus rules
+///
+/// This function uses [`bitcoinconsensus`] to verify transactions by fetching the required data
+/// either from the [`Database`] or using the [`Blockchain`].
+///
+/// Depending on the [capabilities](crate::blockchain::Blockchain::get_capabilities) of the
+/// [`Blockchain`] backend, the method could fail when called with old "historical" transactions or
+/// with unconfirmed transactions that have been evicted from the backend's memory.
+pub fn verify_tx<D: Database, B: Blockchain>(
+    tx: &Transaction,
+    database: &D,
+    blockchain: &B,
+) -> Result<(), VerifyError> {
+    log::debug!("Verifying {}", tx.txid());
+
+    let serialized_tx = serialize(tx);
+    let mut tx_cache = HashMap::<_, Transaction>::new();
+
+    for (index, input) in tx.input.iter().enumerate() {
+        let prev_tx = if let Some(prev_tx) = tx_cache.get(&input.previous_output.txid) {
+            prev_tx.clone()
+        } else if let Some(prev_tx) = database.get_raw_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else if let Some(prev_tx) = blockchain.get_tx(&input.previous_output.txid)? {
+            prev_tx
+        } else {
+            return Err(VerifyError::MissingInputTx(input.previous_output.txid));
+        };
+
+        let spent_output = prev_tx
+            .output
+            .get(input.previous_output.vout as usize)
+            .ok_or(VerifyError::InvalidInput(input.previous_output))?;
+
+        bitcoinconsensus::verify(
+            &spent_output.script_pubkey.to_bytes(),
+            spent_output.value,
+            &serialized_tx,
+            index,
+        )?;
+
+        // Since we have a local cache we might as well cache stuff from the db, as it will very
+        // likely decrease latency compared to reading from disk or performing an SQL query.
+        tx_cache.insert(prev_tx.txid(), prev_tx);
+    }
+
+    Ok(())
+}
+
+/// Error during validation of a tx agains the consensus rules
+#[derive(Debug)]
+pub enum VerifyError {
+    /// The transaction being spent is not available in the database or the blockchain client
+    MissingInputTx(Txid),
+    /// The transaction being spent doesn't have the requested output
+    InvalidInput(OutPoint),
+
+    /// Consensus error
+    Consensus(bitcoinconsensus::Error),
+
+    /// Generic error
+    ///
+    /// It has to be wrapped in a `Box` since `Error` has a variant that contains this enum
+    Global(Box<Error>),
+}
+
+impl fmt::Display for VerifyError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for VerifyError {}
+
+impl From<Error> for VerifyError {
+    fn from(other: Error) -> Self {
+        VerifyError::Global(Box::new(other))
+    }
+}
+impl_error!(bitcoinconsensus::Error, Consensus, VerifyError);
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashSet;
+
+    use bitcoin::consensus::encode::deserialize;
+    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::{Transaction, Txid};
+
+    use crate::blockchain::{Blockchain, Capability, Progress};
+    use crate::database::{BatchDatabase, BatchOperations, MemoryDatabase};
+    use crate::FeeRate;
+
+    use super::*;
+
+    struct DummyBlockchain;
+
+    impl Blockchain for DummyBlockchain {
+        fn get_capabilities(&self) -> HashSet<Capability> {
+            Default::default()
+        }
+        fn setup<D: BatchDatabase, P: 'static + Progress>(
+            &self,
+            _stop_gap: Option<usize>,
+            _database: &mut D,
+            _progress_update: P,
+        ) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_tx(&self, _txid: &Txid) -> Result<Option<Transaction>, Error> {
+            Ok(None)
+        }
+        fn broadcast(&self, _tx: &Transaction) -> Result<(), Error> {
+            Ok(())
+        }
+        fn get_height(&self) -> Result<u32, Error> {
+            Ok(42)
+        }
+        fn estimate_fee(&self, _target: usize) -> Result<FeeRate, Error> {
+            Ok(FeeRate::default_min_relay_fee())
+        }
+    }
+
+    #[test]
+    fn test_verify_fail_unsigned_tx() {
+        // https://blockstream.info/tx/95da344585fcf2e5f7d6cbf2c3df2dcce84f9196f7a7bb901a43275cd6eb7c3f
+        let prev_tx: Transaction = deserialize(&Vec::<u8>::from_hex("020000000101192dea5e66d444380e106f8e53acb171703f00d43fb6b3ae88ca5644bdb7e1000000006b48304502210098328d026ce138411f957966c1cf7f7597ccbb170f5d5655ee3e9f47b18f6999022017c3526fc9147830e1340e04934476a3d1521af5b4de4e98baf49ec4c072079e01210276f847f77ec8dd66d78affd3c318a0ed26d89dab33fa143333c207402fcec352feffffff023d0ac203000000001976a9144bfbaf6afb76cc5771bc6404810d1cc041a6933988aca4b956050000000017a91494d5543c74a3ee98e0cf8e8caef5dc813a0f34b48768cb0700").unwrap()).unwrap();
+        // https://blockstream.info/tx/aca326a724eda9a461c10a876534ecd5ae7b27f10f26c3862fb996f80ea2d45d
+        let signed_tx: Transaction = deserialize(&Vec::<u8>::from_hex("02000000013f7cebd65c27431a90bba7f796914fe8cc2ddfc3f2cbd6f7e5f2fc854534da95000000006b483045022100de1ac3bcdfb0332207c4a91f3832bd2c2915840165f876ab47c5f8996b971c3602201c6c053d750fadde599e6f5c4e1963df0f01fc0d97815e8157e3d59fe09ca30d012103699b464d1d8bc9e47d4fb1cdaa89a1c5783d68363c4dbc4b524ed3d857148617feffffff02836d3c01000000001976a914fc25d6d5c94003bf5b0c7b640a248e2c637fcfb088ac7ada8202000000001976a914fbed3d9b11183209a57999d54d59f67c019e756c88ac6acb0700").unwrap()).unwrap();
+
+        let mut database = MemoryDatabase::new();
+        let blockchain = DummyBlockchain;
+
+        let mut unsigned_tx = signed_tx.clone();
+        for input in &mut unsigned_tx.input {
+            input.script_sig = Default::default();
+            input.witness = Default::default();
+        }
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail with missing input tx");
+        assert!(
+            matches!(result, Err(VerifyError::MissingInputTx(txid)) if txid == prev_tx.txid()),
+            "Error should be a `MissingInputTx` error"
+        );
+
+        // insert the prev_tx
+        database.set_raw_tx(&prev_tx).unwrap();
+
+        let result = verify_tx(&unsigned_tx, &database, &blockchain);
+        assert!(result.is_err(), "Should fail since the TX is unsigned");
+        assert!(
+            matches!(result, Err(VerifyError::Consensus(_))),
+            "Error should be a `Consensus` error"
+        );
+
+        let result = verify_tx(&signed_tx, &database, &blockchain);
+        assert!(
+            result.is_ok(),
+            "Should work since the TX is correctly signed"
+        );
+    }
+}

testutils-macros/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-name = "magical-testutils-macros"
-version = "0.1.0-beta.1"
-authors = ["Alekos Filini <alekos.filini@gmail.com>"]
-edition = "2018"
-
-[lib]
-proc-macro = true
-name = "testutils_macros"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-syn = { version = "1.0", features = ["parsing"] }
-proc-macro2 = "1.0"
-quote = "1.0"
-
-[features]
-debug = ["syn/extra-traits"]

testutils-macros/src/lib.rs

@@ -1,530 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-#[macro_use]
-extern crate quote;
-
-use proc_macro::TokenStream;
-
-use syn::spanned::Spanned;
-use syn::{parse, parse2, Ident, ReturnType};
-
-#[proc_macro_attribute]
-pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenStream {
-    let root_ident = if !attr.is_empty() {
-        match parse::<syn::ExprPath>(attr) {
-            Ok(parsed) => parsed,
-            Err(e) => {
-                let error_string = e.to_string();
-                return (quote! {
-                    compile_error!("Invalid crate path: {:?}", #error_string)
-                })
-                .into();
-            }
-        }
-    } else {
-        parse2::<syn::ExprPath>(quote! { magical }).unwrap()
-    };
-
-    match parse::<syn::ItemFn>(item) {
-        Err(_) => (quote! {
-            compile_error!("#[magical_blockchain_tests] can only be used on `fn`s")
-        })
-        .into(),
-        Ok(parsed) => {
-            let parsed_sig_ident = parsed.sig.ident.clone();
-            let mod_name = Ident::new(
-                &format!("generated_tests_{}", parsed_sig_ident.to_string()),
-                parsed.span(),
-            );
-
-            let return_type = match parsed.sig.output {
-                ReturnType::Type(_, ref t) => t.clone(),
-                ReturnType::Default => {
-                    return (quote! {
-                        compile_error!("The tagged function must return a type that impl `OnlineBlockchain`")
-                    }).into();
-                }
-            };
-
-            let output = quote! {
-
-            #parsed
-
-            mod #mod_name {
-                use bitcoin::Network;
-
-                use miniscript::Descriptor;
-
-                use testutils::{TestClient, serial};
-
-                use #root_ident::blockchain::{OnlineBlockchain, noop_progress};
-                use #root_ident::descriptor::ExtendedDescriptor;
-                use #root_ident::database::MemoryDatabase;
-                use #root_ident::types::ScriptType;
-                use #root_ident::{Wallet, TxBuilder, FeeRate};
-
-                use super::*;
-
-                fn get_blockchain() -> #return_type {
-                    #parsed_sig_ident()
-                }
-
-                fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<#return_type, MemoryDatabase> {
-                    Wallet::new(&descriptors.0.to_string(), descriptors.1.as_deref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
-                }
-
-                fn init_single_sig() -> (Wallet<#return_type, MemoryDatabase>, (String, Option<String>), TestClient) {
-                    let descriptors = testutils! {
-                        @descriptors ( "wpkh(Alice)" ) ( "wpkh(Alice)" ) ( @keys ( "Alice" => (@generate_xprv "/44'/0'/0'/0/*", "/44'/0'/0'/1/*") ) )
-                    };
-
-                    let test_client = TestClient::new();
-                    let wallet = get_wallet_from_descriptors(&descriptors);
-
-                    (wallet, descriptors, test_client)
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_simple() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let tx = testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    };
-                    println!("{:?}", tx);
-                    let txid = test_client.receive(tx);
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_unspent().unwrap()[0].is_internal, false);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.received, 50_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_stop_gap_20() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 5) => 50_000 )
-                    });
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 25) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 100_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_before_and_after_receive() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 0);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_multiple_outputs_same_tx() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000, (@external descriptors, 5) => 30_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 105_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 3);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.received, 105_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_receive_multi() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 5) => 25_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 2);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_address_reuse() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 25_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_receive_rbf_replaced() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) ( @replaceable true )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.received, 50_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-
-                    let new_txid = test_client.bump_fee(&txid);
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, new_txid);
-                    assert_eq!(list_tx_item.received, 50_000);
-                    assert_eq!(list_tx_item.sent, 0);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_reorg_block() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-
-                    let txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) ( @confirmations 1 ) ( @replaceable true )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert!(list_tx_item.height.is_some());
-
-                    // Invalidate 1 block
-                    test_client.invalidate(1);
-
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
-                    assert_eq!(list_tx_item.txid, txid);
-                    assert_eq!(list_tx_item.height, None);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_after_send() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    println!("{}", descriptors.0);
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey(), 25_000)])).unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    let tx = psbt.extract_tx();
-                    println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
-                    wallet.broadcast(tx).unwrap();
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
-                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_outgoing_from_scratch() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    let received_txid = test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey(), 25_000)])).unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    // empty wallet
-                    let wallet = get_wallet_from_descriptors(&descriptors);
-                    wallet.sync(noop_progress(), None).unwrap();
-
-                    let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
-
-                    let received = tx_map.get(&received_txid).unwrap();
-                    assert_eq!(received.received, 50_000);
-                    assert_eq!(received.sent, 0);
-
-                    let sent = tx_map.get(&sent_txid).unwrap();
-                    assert_eq!(sent.received, details.received);
-                    assert_eq!(sent.sent, details.sent);
-                    assert_eq!(sent.fees, details.fees);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_long_change_chain() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 )
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let mut total_sent = 0;
-                    for _ in 0..5 {
-                        let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 5_000)])).unwrap();
-                        let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                        assert!(finalized, "Cannot finalize transaction");
-                        wallet.broadcast(psbt.extract_tx()).unwrap();
-
-                        wallet.sync(noop_progress(), None).unwrap();
-
-                        total_sent += 5_000 + details.fees;
-                    }
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
-
-                    // empty wallet
-                    let wallet = get_wallet_from_descriptors(&descriptors);
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 5_000)]).enable_rbf()).unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fees - 5_000);
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(2.1))).unwrap();
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fees - 5_000);
-                    assert_eq!(wallet.get_balance().unwrap(), new_details.received);
-
-                    assert!(new_details.fees > details.fees);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee_remove_change() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
-
-                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 49_000)]).enable_rbf()).unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fees);
-                    assert_eq!(wallet.get_balance().unwrap(), details.received);
-
-                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(5.0))).unwrap();
-
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 0);
-                    assert_eq!(new_details.received, 0);
-
-                    assert!(new_details.fees > details.fees);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee_add_input() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-
-                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 49_000)]).enable_rbf()).unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                    assert_eq!(details.received, 1_000 - details.fees);
-
-                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(10.0))).unwrap();
-
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(new_details.sent, 75_000);
-                    assert_eq!(wallet.get_balance().unwrap(), new_details.received);
-                }
-
-                #[test]
-                #[serial]
-                fn test_sync_bump_fee_add_input_no_change() {
-                    let (wallet, descriptors, mut test_client) = init_single_sig();
-                    let node_addr = test_client.get_node_address(None);
-
-                    test_client.receive(testutils! {
-                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
-                    });
-
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
-
-                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 49_000)]).enable_rbf()).unwrap();
-                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
-                    assert_eq!(details.received, 1_000 - details.fees);
-
-                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(123.0))).unwrap();
-                    println!("{:#?}", new_details);
-
-                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
-                    assert!(finalized, "Cannot finalize transaction");
-                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
-                    wallet.sync(noop_progress(), None).unwrap();
-                    assert_eq!(new_details.sent, 75_000);
-                    assert_eq!(wallet.get_balance().unwrap(), 0);
-                    assert_eq!(new_details.received, 0);
-                }
-            }
-
-                        };
-
-            output.into()
-        }
-    }
-}

testutils/.gitignore

@@ -1,2 +0,0 @@
-target/
-Cargo.lock

testutils/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "magical-testutils"
-version = "0.1.0-beta.1"
-authors = ["Alekos Filini <alekos.filini@gmail.com>"]
-edition = "2018"
-
-[lib]
-name = "testutils"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-# The latest bitcoincore-rpc depends on an older version of bitcoin, which in turns depends on an
-# older version of secp256k1, which causes conflicts during linking. Use my fork right now, we can
-# switch back to crates.io as soon as rust-bitcoin is updated in rust-bitcoincore-rpc.
-#
-# Tracking issue: https://github.com/rust-bitcoin/rust-bitcoincore-rpc/pull/80
-
-[dependencies]
-log = "0.4.8"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-serial_test = "0.4"
-bitcoin = "0.23"
-bitcoincore-rpc = "0.11"
-electrum-client = "0.2.0-beta.1"

testutils/src/lib.rs

@@ -1,530 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-#[macro_use]
-extern crate serde_json;
-#[macro_use]
-extern crate serial_test;
-
-pub use serial_test::serial;
-
-use std::collections::HashMap;
-use std::env;
-use std::ops::Deref;
-use std::path::PathBuf;
-use std::str::FromStr;
-use std::time::Duration;
-
-#[allow(unused_imports)]
-use log::{debug, error, info, trace};
-
-use bitcoin::consensus::encode::{deserialize, serialize};
-use bitcoin::hashes::hex::{FromHex, ToHex};
-use bitcoin::hashes::sha256d;
-use bitcoin::{Address, Amount, Script, Transaction, Txid};
-
-pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
-pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
-
-pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
-
-// TODO: we currently only support env vars, we could also parse a toml file
-fn get_auth() -> Auth {
-    match env::var("MAGICAL_RPC_AUTH").as_ref().map(String::as_ref) {
-        Ok("USER_PASS") => Auth::UserPass(
-            env::var("MAGICAL_RPC_USER").unwrap(),
-            env::var("MAGICAL_RPC_PASS").unwrap(),
-        ),
-        _ => Auth::CookieFile(PathBuf::from(
-            env::var("MAGICAL_RPC_COOKIEFILE")
-                .unwrap_or("/home/user/.bitcoin/regtest/.cookie".to_string()),
-        )),
-    }
-}
-
-pub fn get_electrum_url() -> String {
-    env::var("MAGICAL_ELECTRUM_URL").unwrap_or("tcp://127.0.0.1:50001".to_string())
-}
-
-pub struct TestClient {
-    client: RpcClient,
-    electrum: ElectrumClient,
-}
-
-#[derive(Clone, Debug)]
-pub struct TestIncomingOutput {
-    pub value: u64,
-    pub to_address: String,
-}
-
-impl TestIncomingOutput {
-    pub fn new(value: u64, to_address: Address) -> Self {
-        Self {
-            value,
-            to_address: to_address.to_string(),
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct TestIncomingTx {
-    pub output: Vec<TestIncomingOutput>,
-    pub min_confirmations: Option<u64>,
-    pub locktime: Option<i64>,
-    pub replaceable: Option<bool>,
-}
-
-impl TestIncomingTx {
-    pub fn new(
-        output: Vec<TestIncomingOutput>,
-        min_confirmations: Option<u64>,
-        locktime: Option<i64>,
-        replaceable: Option<bool>,
-    ) -> Self {
-        Self {
-            output,
-            min_confirmations,
-            locktime,
-            replaceable,
-        }
-    }
-
-    pub fn add_output(&mut self, output: TestIncomingOutput) {
-        self.output.push(output);
-    }
-}
-
-#[macro_export]
-macro_rules! testutils {
-    ( @external $descriptors:expr, $child:expr ) => ({
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
-
-        let parsed = Descriptor::<DescriptorPublicKey>::parse_secret(&$descriptors.0).expect("Failed to parse descriptor in `testutils!(@external)`").0;
-        parsed.derive(&[bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()]).address(bitcoin::Network::Regtest).expect("No address form")
-    });
-    ( @internal $descriptors:expr, $child:expr ) => ({
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
-
-        let parsed = Descriptor::<DescriptorPublicKey>::parse_secret(&$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
-        parsed.derive(&[bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()]).address(bitcoin::Network::Regtest).expect("No address form")
-    });
-    ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
-    ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
-
-    ( @tx ( $( ( $( $addr:tt )* ) => $amount:expr ),+ ) $( ( @locktime $locktime:expr ) )* $( ( @confirmations $confirmations:expr ) )* $( ( @replaceable $replaceable:expr ) )* ) => ({
-        let mut outs = Vec::new();
-        $( outs.push(testutils::TestIncomingOutput::new($amount, testutils!( $($addr)* ))); )+
-
-        let mut locktime = None::<i64>;
-        $( locktime = Some($locktime); )*
-
-        let mut min_confirmations = None::<u64>;
-        $( min_confirmations = Some($confirmations); )*
-
-        let mut replaceable = None::<bool>;
-        $( replaceable = Some($replaceable); )*
-
-        testutils::TestIncomingTx::new(outs, min_confirmations, locktime, replaceable)
-    });
-
-    ( @literal $key:expr ) => ({
-        let key = $key.to_string();
-        (key, None::<String>, None::<String>)
-    });
-    ( @generate_xprv $( $external_path:expr )* $( ,$internal_path:expr )* ) => ({
-        use rand::Rng;
-
-        let mut seed = [0u8; 32];
-        rand::thread_rng().fill(&mut seed[..]);
-
-        let key = bitcoin::util::bip32::ExtendedPrivKey::new_master(
-            bitcoin::Network::Testnet,
-            &seed,
-        );
-
-        let mut external_path = None::<String>;
-        $( external_path = Some($external_path.to_string()); )*
-
-        let mut internal_path = None::<String>;
-        $( internal_path = Some($internal_path.to_string()); )*
-
-        (key.unwrap().to_string(), external_path, internal_path)
-    });
-    ( @generate_wif ) => ({
-        use rand::Rng;
-
-        let mut key = [0u8; bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
-        rand::thread_rng().fill(&mut key[..]);
-
-        (bitcoin::PrivateKey {
-            compressed: true,
-            network: bitcoin::Network::Testnet,
-            key: bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
-        }.to_string(), None::<String>, None::<String>)
-    });
-
-    ( @keys ( $( $alias:expr => ( $( $key_type:tt )* ) ),+ ) ) => ({
-        let mut map = std::collections::HashMap::new();
-        $(
-            let alias: &str = $alias;
-            map.insert(alias, testutils!( $($key_type)* ));
-        )+
-
-        map
-    });
-
-    ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )* $( ( @keys $( $keys:tt )* ) )* ) => ({
-        use std::str::FromStr;
-        use std::collections::HashMap;
-        use std::convert::TryInto;
-
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
-
-        let mut keys: HashMap<&'static str, (String, Option<String>, Option<String>)> = HashMap::new();
-        $(
-            keys = testutils!{ @keys $( $keys )* };
-        )*
-
-        let external: Descriptor<String> = FromStr::from_str($external_descriptor).unwrap();
-        let external: Descriptor<String> = external.translate_pk::<_, _, _, &'static str>(|k| {
-            if let Some((key, ext_path, _)) = keys.get(&k.as_str()) {
-                Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into())))
-            } else {
-                Ok(k.clone())
-            }
-        }, |kh| {
-            if let Some((key, ext_path, _)) = keys.get(&kh.as_str()) {
-                Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into())))
-            } else {
-                Ok(kh.clone())
-            }
-
-        }).unwrap();
-        let external = external.to_string();
-
-        let mut internal = None::<String>;
-        $(
-            let string_internal: Descriptor<String> = FromStr::from_str($internal_descriptor).unwrap();
-
-            let string_internal: Descriptor<String> = string_internal.translate_pk::<_, _, _, &'static str>(|k| {
-                if let Some((key, _, int_path)) = keys.get(&k.as_str()) {
-                    Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into())))
-                } else {
-                    Ok(k.clone())
-                }
-            }, |kh| {
-                if let Some((key, _, int_path)) = keys.get(&kh.as_str()) {
-                    Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into())))
-                } else {
-                    Ok(kh.clone())
-                }
-
-            }).unwrap();
-            internal = Some(string_internal.to_string());
-        )*
-
-        (external, internal)
-    })
-}
-
-fn exponential_backoff_poll<T, F>(mut poll: F) -> T
-where
-    F: FnMut() -> Option<T>,
-{
-    let mut delay = Duration::from_millis(64);
-    loop {
-        match poll() {
-            Some(data) => break data,
-            None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0),
-            None => {}
-        }
-
-        std::thread::sleep(delay);
-    }
-}
-
-impl TestClient {
-    pub fn new() -> Self {
-        let url = env::var("MAGICAL_RPC_URL").unwrap_or("127.0.0.1:18443".to_string());
-        let client = RpcClient::new(format!("http://{}", url), get_auth()).unwrap();
-        let electrum = ElectrumClient::new(&get_electrum_url(), None).unwrap();
-
-        TestClient { client, electrum }
-    }
-
-    fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
-        // wait for electrs to index the tx
-        exponential_backoff_poll(|| {
-            trace!("wait_for_tx {}", txid);
-
-            self.electrum
-                .script_get_history(monitor_script)
-                .unwrap()
-                .iter()
-                .position(|entry| entry.tx_hash == txid)
-        });
-    }
-
-    fn wait_for_block(&mut self, min_height: usize) {
-        self.electrum.block_headers_subscribe().unwrap();
-
-        loop {
-            let header = exponential_backoff_poll(|| {
-                self.electrum.ping().unwrap();
-                self.electrum.block_headers_pop().unwrap()
-            });
-            if header.height >= min_height {
-                break;
-            }
-        }
-    }
-
-    pub fn receive(&mut self, meta_tx: TestIncomingTx) -> Txid {
-        assert!(
-            meta_tx.output.len() > 0,
-            "can't create a transaction with no outputs"
-        );
-
-        let mut map = HashMap::new();
-
-        let mut required_balance = 0;
-        for out in &meta_tx.output {
-            required_balance += out.value;
-            map.insert(out.to_address.clone(), Amount::from_sat(out.value));
-        }
-
-        if self.get_balance(None, None).unwrap() < Amount::from_sat(required_balance) {
-            panic!("Insufficient funds in bitcoind. Plase generate a few blocks with: `bitcoin-cli generatetoaddress 10 {}`", self.get_new_address(None, None).unwrap());
-        }
-
-        // FIXME: core can't create a tx with two outputs to the same address
-        let tx = self
-            .create_raw_transaction_hex(&[], &map, meta_tx.locktime, meta_tx.replaceable)
-            .unwrap();
-        let tx = self.fund_raw_transaction(tx, None, None).unwrap();
-        let mut tx: Transaction = deserialize(&tx.hex).unwrap();
-
-        if let Some(true) = meta_tx.replaceable {
-            // for some reason core doesn't set this field right
-            for input in &mut tx.input {
-                input.sequence = 0xFFFFFFFD;
-            }
-        }
-
-        let tx = self
-            .sign_raw_transaction_with_wallet(&serialize(&tx), None, None)
-            .unwrap();
-
-        // broadcast through electrum so that it caches the tx immediately
-        let txid = self
-            .electrum
-            .transaction_broadcast(&deserialize(&tx.hex).unwrap())
-            .unwrap();
-
-        if let Some(num) = meta_tx.min_confirmations {
-            self.generate(num);
-        }
-
-        let monitor_script = Address::from_str(&meta_tx.output[0].to_address)
-            .unwrap()
-            .script_pubkey();
-        self.wait_for_tx(txid, &monitor_script);
-
-        debug!("Sent tx: {}", txid);
-
-        txid
-    }
-
-    pub fn bump_fee(&mut self, txid: &Txid) -> Txid {
-        let tx = self.get_raw_transaction_info(txid, None).unwrap();
-        assert!(
-            tx.confirmations.is_none(),
-            "Can't bump tx {} because it's already confirmed",
-            txid
-        );
-
-        let bumped: serde_json::Value = self.call("bumpfee", &[txid.to_string().into()]).unwrap();
-        let new_txid = Txid::from_str(&bumped["txid"].as_str().unwrap().to_string()).unwrap();
-
-        let monitor_script =
-            tx.vout[0].script_pub_key.addresses.as_ref().unwrap()[0].script_pubkey();
-        self.wait_for_tx(new_txid, &monitor_script);
-
-        debug!("Bumped {}, new txid {}", txid, new_txid);
-
-        new_txid
-    }
-
-    pub fn generate_manually(&mut self, txs: Vec<Transaction>) -> String {
-        use bitcoin::blockdata::block::{Block, BlockHeader};
-        use bitcoin::blockdata::script::Builder;
-        use bitcoin::blockdata::transaction::{OutPoint, TxIn, TxOut};
-        use bitcoin::hash_types::{BlockHash, TxMerkleNode};
-
-        let block_template: serde_json::Value = self
-            .call("getblocktemplate", &[json!({"rules": ["segwit"]})])
-            .unwrap();
-        trace!("getblocktemplate: {:#?}", block_template);
-
-        let header = BlockHeader {
-            version: block_template["version"].as_u64().unwrap() as u32,
-            prev_blockhash: BlockHash::from_hex(
-                block_template["previousblockhash"].as_str().unwrap(),
-            )
-            .unwrap(),
-            merkle_root: TxMerkleNode::default(),
-            time: block_template["curtime"].as_u64().unwrap() as u32,
-            bits: u32::from_str_radix(block_template["bits"].as_str().unwrap(), 16).unwrap(),
-            nonce: 0,
-        };
-        debug!("header: {:#?}", header);
-
-        let height = block_template["height"].as_u64().unwrap() as i64;
-        let witness_reserved_value: Vec<u8> = sha256d::Hash::default().as_ref().into();
-        // burn block subsidy and fees, not a big deal
-        let mut coinbase_tx = Transaction {
-            version: 1,
-            lock_time: 0,
-            input: vec![TxIn {
-                previous_output: OutPoint::null(),
-                script_sig: Builder::new().push_int(height).into_script(),
-                sequence: 0xFFFFFFFF,
-                witness: vec![witness_reserved_value],
-            }],
-            output: vec![],
-        };
-
-        let mut txdata = vec![coinbase_tx.clone()];
-        txdata.extend_from_slice(&txs);
-
-        let mut block = Block { header, txdata };
-
-        let witness_root = block.witness_root();
-        let witness_commitment =
-            Block::compute_witness_commitment(&witness_root, &coinbase_tx.input[0].witness[0]);
-
-        // now update and replace the coinbase tx
-        let mut coinbase_witness_commitment_script = vec![0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed];
-        coinbase_witness_commitment_script.extend_from_slice(&witness_commitment);
-
-        coinbase_tx.output.push(TxOut {
-            value: 0,
-            script_pubkey: coinbase_witness_commitment_script.into(),
-        });
-        block.txdata[0] = coinbase_tx;
-
-        // set merkle root
-        let merkle_root = block.merkle_root();
-        block.header.merkle_root = merkle_root;
-
-        assert!(block.check_merkle_root());
-        assert!(block.check_witness_commitment());
-
-        // now do PoW :)
-        let target = block.header.target();
-        while block.header.validate_pow(&target).is_err() {
-            block.header.nonce = block.header.nonce.checked_add(1).unwrap(); // panic if we run out of nonces
-        }
-
-        let block_hex: String = serialize(&block).to_hex();
-        debug!("generated block hex: {}", block_hex);
-
-        self.electrum.block_headers_subscribe().unwrap();
-
-        let submit_result: serde_json::Value =
-            self.call("submitblock", &[block_hex.into()]).unwrap();
-        debug!("submitblock: {:?}", submit_result);
-        assert!(
-            submit_result.is_null(),
-            "submitblock error: {:?}",
-            submit_result.as_str()
-        );
-
-        self.wait_for_block(height as usize);
-
-        block.header.block_hash().to_hex()
-    }
-
-    pub fn generate(&mut self, num_blocks: u64) {
-        let our_addr = self.get_new_address(None, None).unwrap();
-        let hashes = self.generate_to_address(num_blocks, &our_addr).unwrap();
-        let best_hash = hashes.last().unwrap();
-        let height = self.get_block_info(best_hash).unwrap().height;
-
-        self.wait_for_block(height);
-
-        debug!("Generated blocks to new height {}", height);
-    }
-
-    pub fn invalidate(&mut self, num_blocks: u64) {
-        self.electrum.block_headers_subscribe().unwrap();
-
-        let best_hash = self.get_best_block_hash().unwrap();
-        let initial_height = self.get_block_info(&best_hash).unwrap().height;
-
-        let mut to_invalidate = best_hash;
-        for i in 1..=num_blocks {
-            trace!(
-                "Invalidating block {}/{} ({})",
-                i,
-                num_blocks,
-                to_invalidate
-            );
-
-            self.invalidate_block(&to_invalidate).unwrap();
-            to_invalidate = self.get_best_block_hash().unwrap();
-        }
-
-        self.wait_for_block(initial_height - num_blocks as usize);
-
-        debug!(
-            "Invalidated {} blocks to new height of {}",
-            num_blocks,
-            initial_height - num_blocks as usize
-        );
-    }
-
-    pub fn reorg(&mut self, num_blocks: u64) {
-        self.invalidate(num_blocks);
-        self.generate(num_blocks);
-    }
-
-    pub fn get_node_address(&self, address_type: Option<AddressType>) -> Address {
-        Address::from_str(
-            &self
-                .get_new_address(None, address_type)
-                .unwrap()
-                .to_string(),
-        )
-        .unwrap()
-    }
-}
-
-impl Deref for TestClient {
-    type Target = RpcClient;
-
-    fn deref(&self) -> &Self::Target {
-        &self.client
-    }
-}