Bump version to 0.2.1-dev

The syntax now is pretty much the same as the normal descriptor syntax,
with the only difference that modifiers cannot be grouped together (i.e.
`sdv:older(144)` must be turned into `s:d:v:older(144)`.

.github/pull_request_template.md

@@ -0,0 +1,30 @@
+<!-- You can erase any parts of this template not applicable to your Pull Request. -->
+
+### Description
+
+<!-- Describe the purpose of this PR, what's being adding and/or fixed -->
+
+### Notes to the reviewers
+
+<!-- In this section you can include notes directed to the reviewers, like explaining why some parts
+of the PR were done in a specific way -->
+
+### Checklists
+
+#### All Submissions:
+
+* [ ] I've signed all my commits
+* [ ] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
+* [ ] I ran `cargo fmt` and `cargo clippy` before committing
+
+#### New Features:
+
+* [ ] I've added tests for the new feature
+* [ ] I've added docs for the new feature
+* [ ] I've updated `CHANGELOG.md`
+
+#### Bugfixes:
+
+* [ ] This pull request breaks the existing API
+* [ ] I've added tests to reproduce the issue which are now passing
+* [ ] I'm linking the issue being fixed by this PR

.github/workflows/audit.yml

@@ -0,0 +1,19 @@
+name: Audit
+
+on:
+  push:
+    paths:
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+  schedule:
+    - cron: '0 0 * * 0' # Once per week
+
+jobs:
+
+  security_audit:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/code_coverage.yml

@@ -0,0 +1,27 @@
+on: [push]
+
+name: Code Coverage
+
+jobs:
+  tarpaulin-codecov:
+    name: Tarpaulin to codecov.io
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+
+      - name: Install tarpaulin
+        run: cargo install cargo-tarpaulin
+      - name: Tarpaulin
+        run: cargo tarpaulin --features all-keys,cli-utils,compiler,esplora,compact_filters --run-types Tests,Doctests --exclude-files "testutils/*" --out Xml
+
+      - name: Publish to codecov.io
+        uses: codecov/codecov-action@v1.0.15
+        with:
+          fail_ci_if_error: true
+          file: ./cobertura.xml

.github/workflows/cont_integration.yml

@@ -0,0 +1,148 @@
+on: [push, pull_request]
+
+name: CI
+
+jobs:
+
+  build-test:
+    name: Build and test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+          - 1.45.0 # MSRV
+        features:
+          - default
+          - minimal
+          - all-keys
+          - minimal,esplora
+          - key-value-db
+          - electrum
+          - compact_filters
+          - cli-utils,esplora,key-value-db,electrum
+          - compiler
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Generate cache key
+        run: echo "${{ matrix.rust }} ${{ matrix.features }}" | tee .cache_key
+      - name: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('.cache_key') }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Set default toolchain
+        run: rustup default ${{ matrix.rust }}
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add clippy
+        run: rustup component add clippy
+      - name: Build
+        run: cargo build --features ${{ matrix.features }} --no-default-features
+      - name: Clippy
+        run: cargo clippy -- -D warnings
+      - name: Test
+        run: cargo test --features ${{ matrix.features }} --no-default-features
+
+  test-readme-examples:
+    name: Test README.md examples
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-test-md-docs-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Test
+        run: cargo test --features test-md-docs --no-default-features -- doctest::ReadmeDoctests
+
+  test-electrum:
+    name: Test electrum
+    runs-on: ubuntu-16.04
+    container: bitcoindevkit/electrs
+    env:
+      MAGICAL_RPC_AUTH: USER_PASS
+      MAGICAL_RPC_USER: admin
+      MAGICAL_RPC_PASS: passw
+      MAGICAL_RPC_URL: 127.0.0.1:18443
+      MAGICAL_ELECTRUM_URL: tcp://127.0.0.1:60401
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Install rustup
+        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - name: Set default toolchain
+        run: $HOME/.cargo/bin/rustup default stable
+      - name: Set profile
+        run: $HOME/.cargo/bin/rustup set profile minimal
+      - name: Start core
+        run: ./ci/start-core.sh
+      - name: Test
+        run: $HOME/.cargo/bin/cargo test --features test-electrum --no-default-features
+
+  check-wasm:
+    name: Check WASM
+    runs-on: ubuntu-16.04
+    env:
+      CC: clang-10
+      CFLAGS: -I/usr/include
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+        # Install a recent version of clang that supports wasm32
+      - run: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
+      - run: sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" || exit 1
+      - run: sudo apt-get update || exit 1
+      - run: sudo apt-get install -y clang-10 libc6-dev-i386 || exit 1
+      - name: Set default toolchain
+        run: rustup default stable
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add target wasm32
+        run: rustup target add wasm32-unknown-unknown
+      - name: Check
+        run: cargo check --target wasm32-unknown-unknown --features cli-utils,esplora --no-default-features
+
+  fmt:
+    name: Rust fmt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set default toolchain
+        run: rustup default stable
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add clippy
+        run: rustup component add rustfmt
+      - name: Check fmt
+        run: cargo fmt --all -- --check

.github/workflows/nightly_docs.yml

@@ -0,0 +1,64 @@
+name: Publish Nightly Docs
+
+on: [push, pull_request]
+
+jobs:
+  build_docs:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Setup cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: nightly-docs-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Install nightly toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+      - name: Build docs
+        uses: actions-rs/cargo@v1
+        with:
+          command: rustdoc
+          args: --verbose --features=compiler,electrum,esplora,compact_filters,key-value-db,all-keys -- --cfg docsrs
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: built-docs
+          path: ./target/doc/*
+
+  publish_docs:
+    name: 'Publish docs'
+    if: github.ref == 'refs/heads/master'
+    needs: [build_docs]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout `bitcoindevkit.org`
+        uses: actions/checkout@v2
+        with:
+          ssh-key: ${{ secrets.DOCS_PUSH_SSH_KEY }}
+          repository: bitcoindevkit/bitcoindevkit.org
+          ref: master
+      - name: Create directories
+        run: mkdir -p ./static/docs-rs/bdk/nightly
+      - name: Remove old latest
+        run: rm -rf ./static/docs-rs/bdk/nightly/latest
+      - name: Download built docs
+        uses: actions/download-artifact@v1
+        with:
+          name: built-docs
+          path: ./static/docs-rs/bdk/nightly/latest
+      - name: Configure git
+        run: git config user.email "github-actions@github.com" && git config user.name "github-actions"
+      - name: Commit
+        continue-on-error: true         # If there's nothing to commit this step fails, but it's fine
+        run: git add ./static && git commit -m "Publish autogenerated nightly docs"
+      - name: Push
+        run: git push origin master

.gitignore

@@ -2,3 +2,4 @@
 Cargo.lock
 
 *.swp
+.idea

.travis.yml

@@ -1,65 +0,0 @@
-language: rust
-rust:
-  - stable
-
-env:
-  global:
-    - MAGICAL_RPC_COOKIEFILE=/home/travis/.bitcoin/regtest/.cookie
-    - MAGICAL_ELECTRUM_URL=tcp://127.0.0.1:60401
-  jobs:
-    - TARGET=x86_64-unknown-linux-gnu    CHECK_FMT=1
-    - TARGET=x86_64-unknown-linux-gnu    RUN_TESTS=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=minimal            NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=minimal,esplora    NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=key-value-db       NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=electrum           NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=compact_filters    NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=cli-utils,esplora  NO_DEFAULT_FEATURES=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=compiler           NO_DEFAULT_FEATURES=1    RUN_TESTS=1 # Test the `miniscriptc` example
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=test-electrum      NO_DEFAULT_FEATURES=1    RUN_TESTS=1    RUN_CORE=1
-    - TARGET=x86_64-unknown-linux-gnu    FEATURES=test-md-docs       NO_DEFAULT_FEATURES=1    RUN_TESTS=1    NIGHTLY=1
-    - TARGET=wasm32-unknown-unknown      FEATURES=cli-utils,esplora  NO_DEFAULT_FEATURES=1
-before_script:
-  - |
-    if [[ "$TARGET" = "wasm32-unknown-unknown" ]]; then
-      # Install a recent version of clang that supports wasm32
-      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
-      sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" || exit 1
-      sudo apt-get update || exit 1
-      sudo apt-get install -y clang-10 libc6-dev-i386 || exit 1
-      export CC="clang-10"
-      export CFLAGS="-I/usr/include"
-    fi
-  - |
-    if [[ $CHECK_FMT -eq 1 ]]; then
-      rustup component add rustfmt
-    fi
-  - |
-    if [[ $NIGHTLY -eq 1 ]]; then
-      rustup toolchain install nightly
-      rustup default nightly
-    fi
-  - rustup target add "$TARGET"
-script:
-  - |
-    if [[ $CHECK_FMT -eq 1 ]]; then
-      cargo fmt -- --check || exit 1
-    fi
-  - |
-    if [[ $RUN_TESTS -eq 1 ]]; then
-        CMD=test
-    else
-        CMD=build
-    fi
-  - |
-    if [[ $RUN_CORE -eq 1 ]]; then
-        ./ci/start-core.sh || exit 1
-    fi
-  - cargo $CMD --verbose --target=$TARGET --features=$FEATURES $( (( NO_DEFAULT_FEATURES == 1 )) && printf %s '--no-default-features' )
-
-notifications:
-  email: false
-
-before_cache:
-  - rm -rf "$TRAVIS_HOME/.cargo/registry/src"
-cache: cargo

CHANGELOG.md

@@ -0,0 +1,213 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Project
+#### Added
+- Add CONTRIBUTING.md
+- Add a Discord badge to the README
+- Add code coverage github actions workflow
+- Add scheduled audit check in CI
+- Add CHANGELOG.md
+
+#### Changed
+- Rename the library to `bdk`
+- Rename `ScriptType` to `KeychainKind`
+- Prettify README examples on github
+- Change CI to github actions
+- Bump rust-bitcoin to 0.25, fix Cargo dependencies
+- Enable clippy for stable and tests by default
+- Switch to "mainline" rust-miniscript
+- Generate a different cache key for every CI job
+- Fix to at least bitcoin ^0.25.2
+
+#### Fixed
+- Fix or ignore clippy warnings for all optional features except compact_filters
+- Pin cc version because last breaks rocksdb build
+
+### Blockchain
+#### Added
+- Add a trait to create `Blockchain`s from a configuration
+- Add an `AnyBlockchain` enum to allow switching at runtime
+- Document `AnyBlockchain` and `ConfigurableBlockchain`
+- Use our Instant struct to be compatible with wasm
+- Make esplora call in parallel
+- Allow to set concurrency in Esplora config and optionally pass it in repl
+
+#### Fixed
+- Fix receiving a coinbase using Electrum/Esplora
+- Use proper type for EsploraHeader, make conversion to BlockHeader infallible
+- Eagerly unwrap height option, save one collect
+
+#### Changed
+- Simplify the architecture of blockchain traits
+- Improve sync
+- Remove unused varaint HeaderParseFail
+
+### CLI
+#### Added
+- Conditionally remove cli args according to enabled feature
+
+#### Changed
+- Add max_addresses param in sync
+- Split the internal and external policy paths
+
+### Database
+#### Added
+- Add `AnyDatabase` and `ConfigurableDatabase` traits
+
+### Descriptor
+#### Added
+- Add a macro to write descriptors from code
+- Add descriptor templates, add `DerivableKey`
+- Add ToWalletDescriptor trait tests
+- Add support for `sortedmulti` in `descriptor!`
+- Add ExtractPolicy trait tests
+- Add get_checksum tests, cleanup tests
+- Add descriptor macro tests
+
+#### Changes
+- Improve the descriptor macro, add traits for key and descriptor types
+
+#### Fixes
+- Fix the recovery of a descriptor given a PSBT
+
+### Keys
+#### Added
+- Add BIP39 support
+- Take `ScriptContext` into account when converting keys
+- Add a way to restrict the networks in which keys are valid
+- Add a trait for keys that can be generated
+- Fix entropy generation
+- Less convoluted entropy generation
+- Re-export tiny-bip39
+- Implement `GeneratableKey` trait for `bitcoin::PrivateKey`
+- Implement `ToDescriptorKey` trait for `GeneratedKey`
+- Add a shortcut to generate keys with the default options
+
+#### Fixed
+- Fix all-keys and cli-utils tests
+
+### Wallet
+#### Added
+- Allow to define static fees for transactions Fixes #137
+- Merging two match expressions for fee calculation
+- Incorporate RBF rules into utxo selection function
+- Add Branch and Bound coin selection
+- Add tests for BranchAndBoundCoinSelection::coin_select
+- Add tests for BranchAndBoundCoinSelection::bnb
+- Add tests for BranchAndBoundCoinSelection::single_random_draw
+- Add test that shwpkh populates witness_utxo
+- Add witness and redeem scripts to PSBT outputs
+- Add an option to include `PSBT_GLOBAL_XPUB`s in PSBTs
+- Eagerly finalize inputs
+
+#### Changed
+- Use collect to avoid iter unwrapping Options
+- Make coin_select take may/must use utxo lists
+- Improve `CoinSelectionAlgorithm`
+- Refactor `Wallet::bump_fee()`
+- Default to SIGHASH_ALL if not specified
+- Replace ChangeSpendPolicy::filter_utxos with a predicate
+- Make 'unspendable' into a HashSet
+- Stop implicitly enforcing manaul selection by .add_utxo
+- Rename DumbCS to LargestFirstCoinSelection
+- Rename must_use_utxos to required_utxos
+- Rename may_use_utxos to optional_uxtos
+- Rename get_must_may_use_utxos to preselect_utxos
+- Remove redundant Box around address validators
+- Remove redundant Box around signers
+- Make Signer and AddressValidator Send and Sync
+- Split `send_all` into `set_single_recipient` and `drain_wallet`
+- Use TXIN_DEFAULT_WEIGHT constant in coin selection
+- Replace `must_use` with `required` in coin selection
+- Take both spending policies into account in create_tx
+- Check last derivation in cache to avoid recomputation
+- Use the branch-and-bound cs by default
+- Make coin_select return UTXOs instead of TxIns
+- Build output lookup inside complete transaction
+- Don't wrap SignersContainer arguments in Arc
+- More consistent references with 'signers' variables
+
+#### Fixed
+- Fix signing for `ShWpkh` inputs
+- Fix the recovery of a descriptor given a PSBT
+
+### Examples
+#### Added
+- Support esplora blockchain source in repl
+
+#### Changed
+- Revert back the REPL example to use Electrum
+- Remove the `magic` alias for `repl`
+- Require esplora feature for repl example
+
+#### Security
+- Use dirs-next instead of dirs since the latter is unmantained
+
+## [0.1.0-beta.1] - 2020-09-08
+
+### Blockchain
+#### Added
+- Lightweight Electrum client with SSL/SOCKS5 support
+- Add a generalized "Blockchain" interface
+- Add Error::OfflineClient
+- Add the Esplora backend
+- Use async I/O in the various blockchain impls
+- Compact Filters blockchain implementation
+- Add support for Tor
+- Impl OnlineBlockchain for types wrapped in Arc
+
+### Database
+#### Added
+- Add a generalized database trait and a Sled-based implementation
+- Add an in-memory database
+
+### Descriptor
+#### Added
+- Wrap Miniscript descriptors to support xpubs
+- Policy and contribution
+- Transform a descriptor into its "public" version
+- Use `miniscript::DescriptorPublicKey`
+
+### Macros
+#### Added
+- Add a feature to enable the async interface on non-wasm32 platforms
+
+### Wallet
+#### Added
+- Wallet logic
+- Add `assume_height_reached` in PSBTSatisfier
+- Add an option to change the assumed current height
+- Specify the policy branch with a map
+- Add a few commands to handle psbts
+- Add hd_keypaths to outputs
+- Add a `TxBuilder` struct to simplify `create_tx()`'s interface
+- Abstract coin selection in a separate trait
+- Refill the address pool whenever necessary
+- Implement the wallet import/export format from FullyNoded
+- Add a type convert fee units, add `Wallet::estimate_fee()`
+- TxOrdering, shuffle/bip69 support
+- Add RBF and custom versions in TxBuilder
+- Allow limiting the use of internal utxos in TxBuilder
+- Add `force_non_witness_utxo()` to TxBuilder
+- RBF and add a few tests
+- Add AddressValidators
+- Add explicit ordering for the signers
+- Support signing the whole tx instead of individual inputs
+- Create a PSBT signer from an ExtendedDescriptor
+
+### Examples
+#### Added
+- Add REPL broadcast command
+- Add a miniscript compiler CLI
+- Expose list_transactions() in the REPL
+- Use `MemoryDatabase` in the compiler example
+- Make the REPL return JSON
+
+[unreleased]: https://github.com/bitcoindevkit/bdk/compare/0.1.0-beta.1...HEAD
+[0.1.0-beta.1]: https://github.com/bitcoindevkit/bdk/compare/96c87ea5...0.1.0-beta.1

CONTRIBUTING.md

@@ -0,0 +1,103 @@
+Contributing to BDK
+==============================
+
+The BDK project operates an open contributor model where anyone is welcome to
+contribute towards development in the form of peer review, documentation,
+testing and patches.
+
+Anyone is invited to contribute without regard to technical experience,
+"expertise", OSS experience, age, or other concern. However, the development of
+cryptocurrencies demands a high-level of rigor, adversarial thinking, thorough
+testing and risk-minimization.
+Any bug may cost users real money. That being said, we deeply welcome people
+contributing for the first time to an open source project or pick up Rust while
+contributing. Don't be shy, you'll learn.
+
+Communications Channels
+-----------------------
+
+Communication about BDK happens primarily on the [BDK Discord](https://discord.gg/dstn4dQ).
+
+Discussion about code base improvements happens in GitHub [issues](https://github.com/bitcoindevkit/bdk/issues) and
+on [pull requests](https://github.com/bitcoindevkit/bdk/pulls).
+
+Contribution Workflow
+---------------------
+
+The codebase is maintained using the "contributor workflow" where everyone
+without exception contributes patch proposals using "pull requests". This
+facilitates social contribution, easy testing and peer review.
+
+To contribute a patch, the worflow is a as follows:
+
+  1. Fork Repository
+  2. Create topic branch
+  3. Commit patches
+
+In general commits should be atomic and diffs should be easy to read.
+For this reason do not mix any formatting fixes or code moves with actual code
+changes. Further, each commit, individually, should compile and pass tests, in
+order to ensure git bisect and other automated tools function properly.
+
+When adding a new feature, thought must be given to the long term technical
+debt.
+Every new feature should be covered by functional tests where possible.
+
+When refactoring, structure your PR to make it easy to review and don't
+hesitate to split it into multiple small, focused PRs.
+
+The Minimal Supported Rust Version is 1.45 (enforced by our CI).
+
+Commits should cover both the issue fixed and the solution's rationale.
+These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind.
+
+To facilitate communication with other contributors, the project is making use
+of GitHub's "assignee" field. First check that no one is assigned and then
+comment suggesting that you're working on it. If someone is already assigned,
+don't hesitate to ask if the assigned party or previous commenters are still
+working on it if it has been awhile.
+
+Peer review
+-----------
+
+Anyone may participate in peer review which is expressed by comments in the
+pull request. Typically reviewers will review the code for obvious errors, as
+well as test out the patch set and opine on the technical merits of the patch.
+PR should be reviewed first on the conceptual level before focusing on code
+style or grammar fixes.
+
+Coding Conventions
+------------------
+
+This codebase uses spaces, not tabs.
+Use `cargo fmt` with the default settings to format code before committing.
+This is also enforced by the CI.
+
+Security
+--------
+
+Security is a high priority of BDK; disclosure of security vulnerabilites helps
+prevent user loss of funds.
+
+Note that BDK is currently considered "pre-production" during this time, there
+is no special handling of security issues. Please simply open an issue on
+Github.
+
+Testing
+-------
+
+Related to the security aspect, BDK developers take testing very seriously.
+Due to the modular nature of the project, writing new functional tests is easy
+and good test coverage of the codebase is an important goal.
+Refactoring the project to enable fine-grained unit testing is also an ongoing
+effort.
+
+Going further
+-------------
+
+You may be interested by Jon Atacks guide on [How to review Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-review-bitcoin-core-prs.md)
+and [How to make Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-make-bitcoin-core-prs.md).
+While there are differences between the projects in terms of context and
+maturity, many of the suggestions offered apply to this project.
+
+Overall, have fun :)

Cargo.toml

@@ -1,33 +1,40 @@
 [package]
-name = "magical"
-version = "0.1.0"
+name = "bdk"
+version = "0.2.1-dev"
 edition = "2018"
 authors = ["Alekos Filini <alekos.filini@gmail.com>", "Riccardo Casatta <riccardo@casatta.it>"]
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk"
+description = "A modern, lightweight, descriptor-based wallet library"
+keywords = ["bitcoin", "wallet", "descriptor", "psbt"]
+readme = "README.md"
+license-file = "LICENSE"
 
 [dependencies]
-magical-macros = { version = "0.1.0-beta.1", path = "./macros" }
+bdk-macros = "0.2"
 log = "^0.4"
-bitcoin = { version = "0.23", features = ["use-serde"] }
-miniscript = { version = "1.0" }
+miniscript = "4.0"
+bitcoin = { version = "^0.25.2", features = ["use-serde"] }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
 rand = "^0.7"
 
 # Optional dependencies
 sled = { version = "0.34", optional = true }
-electrum-client = { version = "0.2.0-beta.1", optional = true }
+electrum-client = { version = "0.4.0-beta.1", optional = true }
 reqwest = { version = "0.10", optional = true, features = ["json"] }
 futures = { version = "0.3", optional = true }
 clap = { version = "2.33", optional = true }
 base64 = { version = "^0.11", optional = true }
 async-trait = { version = "0.1", optional = true }
 rocksdb = { version = "0.14", optional = true }
+# pin cc version to 1.0.62 because 1.0.63 break rocksdb build
+cc = { version = "=1.0.62", optional = true }
 socks = { version = "0.3", optional = true }
 lazy_static = { version = "1.4", optional = true }
-
-[patch.crates-io]
-bitcoin = { git = "https://github.com/rust-bitcoin/rust-bitcoin/", rev = "478e091" }
-miniscript = { git = "https://github.com/MagicalBitcoin/rust-miniscript", branch = "descriptor-public-key" }
+tiny-bip39 = { version = "^0.8", optional = true }
+structopt = { version = "^0.3", optional = true }
 
 # Platform-specific dependencies
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -36,6 +43,7 @@ tokio = { version = "0.2", features = ["rt-core"] }
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 async-trait = "0.1"
 js-sys = "0.3"
+rand = { version = "^0.7", features = ["wasm-bindgen"] }
 
 [features]
 minimal = []
@@ -43,25 +51,26 @@ compiler = ["clap", "miniscript/compiler"]
 default = ["key-value-db", "electrum"]
 electrum = ["electrum-client"]
 esplora = ["reqwest", "futures"]
-compact_filters = ["rocksdb", "socks", "lazy_static"]
+compact_filters = ["rocksdb", "socks", "lazy_static", "cc"]
 key-value-db = ["sled"]
-cli-utils = ["clap", "base64"]
+cli-utils = ["clap", "base64", "structopt"]
 async-interface = ["async-trait"]
+all-keys = ["keys-bip39"]
+keys-bip39 = ["tiny-bip39"]
 
 # Debug/Test features
-debug-proc-macros = ["magical-macros/debug", "magical-testutils-macros/debug"]
+debug-proc-macros = ["bdk-macros/debug", "bdk-testutils-macros/debug"]
 test-electrum = ["electrum"]
 test-md-docs = ["base64", "electrum"]
 
 [dev-dependencies]
-magical-testutils = { version = "0.1.0-beta.1", path = "./testutils" }
-magical-testutils-macros = { version = "0.1.0-beta.1", path = "./testutils-macros" }
+bdk-testutils = "0.2"
+bdk-testutils-macros = "0.2"
 serial_test = "0.4"
 lazy_static = "1.4"
 rustyline = "6.0"
-dirs = "2.0"
+dirs-next = "2.0"
 env_logger = "0.7"
-rand = "0.7"
 
 [[example]]
 name = "repl"
@@ -76,18 +85,12 @@ name = "miniscriptc"
 path = "examples/compiler.rs"
 required-features = ["compiler"]
 
-# Provide a more user-friendly alias for the REPL
-[[example]]
-name = "magic"
-path = "examples/repl.rs"
-required-features = ["cli-utils"]
-
 [workspace]
 members = ["macros", "testutils", "testutils-macros"]
 
 # Generate docs with nightly to add the "features required" badge
 # https://stackoverflow.com/questions/61417452/how-to-get-a-feature-requirement-tag-in-the-documentation-generated-by-cargo-do
 [package.metadata.docs.rs]
-features = ["compiler", "electrum", "esplora", "compact_filters", "key-value-db"]
+features = ["compiler", "electrum", "esplora", "compact_filters", "key-value-db", "all-keys"]
 # defines the configuration attribute `docsrs`
 rustdoc-args = ["--cfg", "docsrs"]

DEVELOPMENT_CYCLE.md

@@ -0,0 +1,46 @@
+# Development Cycle
+
+This project follows a regular releasing schedule similar to the one [used by the Rust language](https://doc.rust-lang.org/book/appendix-07-nightly-rust.html). In short, this means that a new release is made at a regular
+cadence, with all the feature/bugfixes that made it to `master` in time. This ensures that we don't keep delaying releases waiting for "just one more little thing".
+
+We decided to maintain a faster release cycle while the library is still in "beta", i.e. before release `1.0.0`: since we are constantly adding new features and, even more importantly, fixing issues, we want developers
+to have access to those updates as fast as possible. For this reason we will make a release **every 4 weeks**.
+
+Once the project will have reached a more mature state (>= `1.0.0`), we will very likely switch to longer release cycles of **6 weeks**.
+
+The "feature freeze" will happen **one week before the release date**. This means a new branch will be created originating from the `master` tip at that time, and in that branch we will stop adding new features and only focus
+on ensuring the ones we've added are working properly.
+
+```
+master:           - - - - * - - - * - - - - - - * - - - * ...
+                          |      /              |       |
+release/0.x.0:            * - - #               |       |
+                                                |      /
+release/0.y.0:                                  * - - #
+```
+
+As soon as the release is tagged and published, the `release` branch will be merged back into `master` to update the version in the `Cargo.toml` to apply the new `Cargo.toml` version and all the other fixes made during the feature
+freeze window.
+
+## Making the Release
+
+What follows are notes and procedures that maintaners can refer to when making releases. All the commits and tags must be signed and, ideally, also [timestamped](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md).
+
+Pre-`v1.0.0` our "major" releases only affect the "minor" semver value. Accordingly, our "minor" releases will only affect the "patch" value.
+
+1. Create a new branch called `release/x.y.z` from `master`. Double check that your local `master` is up-to-date with the upstream repo before doing so.
+2. Make a commit on the release branch to bump the version to `x.y.z-rc.1`. The message should be "Bump version to x.y.z-rc.1".
+3. Push the new branch to `bitcoindevkit/bdk` on GitHub.
+4. During the one week of feature freeze run additional tests on the release branch
+5. If a bug is found:
+    - If it's a minor issue you can just fix it in the release branch, since it will be merged back to `master` eventually
+    - For bigger issues you can fix them on `master` and then *cherry-pick* the commit to the release branch
+6. On release day, make a commit on the release branch to bump the version to `x.y.z`. The message should be "Bump version to x.y.z".
+7. Add a tag to this commit. The tag name should be `vx.y.z` (for example `v0.5.0`), and the message "Release x.y.z". Make sure the tag is signed, for extra safety use the explicit `--sign` flag.
+8. Push the new commits to the upstream release branch, wait for the CI to finish one last time.
+9. Publish **all** the updated crates to crates.io.
+10. Make a new commit to bump the version value to `x.y.(z+1)-dev`. The message should be "Bump version to x.y.(z+1)-dev".
+11. Merge the release branch back into `master`.
+12. Make sure the new release shows up on crates.io and that the docs are built correctly on docs.rs.
+13. Announce the release on Twitter, Discord and Telegram.
+14. Celebrate :tada:

README.md

@@ -1,7 +1,7 @@
 <div align="center">
-  <h1>Magical Bitcoin Library</h1>
+  <h1>BDK</h1>
 
-  <img src="./static/wizard.svg" width="220" />
+  <img src="./static/bdk.svg" width="220" />
 
   <p>
     <strong>A modern, lightweight, descriptor-based wallet library written in Rust!</strong>
@@ -9,21 +9,23 @@
 
   <p>
     <!-- <a href="https://crates.io/crates/magical"><img alt="Crate Info" src="https://img.shields.io/crates/v/magical.svg"/></a> -->
-    <a href="https://travis-ci.org/MagicalBitcoin/magical-bitcoin-wallet"><img alt="Traivs Status" src="https://travis-ci.org/MagicalBitcoin/magical-bitcoin-wallet.svg?branch=master"></a>
-    <a href="https://magicalbitcoin.org/docs-rs/magical"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-magical-green"/></a>
+    <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
+    <a href="https://codecov.io/gh/bitcoindevkit/bdk"><img src="https://codecov.io/gh/bitcoindevkit/bdk/branch/master/graph/badge.svg"/></a>
+    <a href="https://bitcoindevkit.org/docs-rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
     <a href="https://blog.rust-lang.org/2020/07/16/Rust-1.45.0.html"><img alt="Rustc Version 1.45+" src="https://img.shields.io/badge/rustc-1.45%2B-lightgrey.svg"/></a>
+    <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
 
   <h4>
-    <a href="https://magicalbitcoin.org">Project Homepage</a>
+    <a href="https://bitcoindevkit.org">Project Homepage</a>
     <span> | </span>
-    <a href="https://magicalbitcoin.org/docs-rs/magical">Documentation</a>
+    <a href="https://bitcoindevkit.org/docs-rs/bdk">Documentation</a>
   </h4>
 </div>
 
 ## About
 
-The `magical` library aims to be the core building block for Bitcoin wallets of any kind.
+The `bdk` library aims to be the core building block for Bitcoin wallets of any kind.
 
 * It uses [Miniscript](https://github.com/rust-bitcoin/rust-miniscript) to support descriptors with generalized conditions. This exact same library can be used to build
   single-sig wallets, multisigs, timelocked contracts and more.
@@ -35,15 +37,15 @@ The `magical` library aims to be the core building block for Bitcoin wallets of
 
 ### Sync the balance of a descriptor
 
-```no_run
-use magical::Wallet;
-use magical::database::MemoryDatabase;
-use magical::blockchain::{noop_progress, ElectrumBlockchain};
+```rust,no_run
+use bdk::Wallet;
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
 
-use magical::electrum_client::Client;
+use bdk::electrum_client::Client;
 
-fn main() -> Result<(), magical::Error> {
-    let client = Client::new("ssl://electrum.blockstream.info:60002", None)?;
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
     let wallet = Wallet::new(
         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
@@ -62,11 +64,11 @@ fn main() -> Result<(), magical::Error> {
 
 ### Generate a few addresses
 
-```
-use magical::{Wallet, OfflineWallet};
-use magical::database::MemoryDatabase;
+```rust
+use bdk::{Wallet, OfflineWallet};
+use bdk::database::MemoryDatabase;
 
-fn main() -> Result<(), magical::Error> {
+fn main() -> Result<(), bdk::Error> {
     let wallet: OfflineWallet<_> = Wallet::new_offline(
         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
@@ -84,17 +86,17 @@ fn main() -> Result<(), magical::Error> {
 
 ### Create a transaction
 
-```no_run
-use magical::{FeeRate, TxBuilder, Wallet};
-use magical::database::MemoryDatabase;
-use magical::blockchain::{noop_progress, ElectrumBlockchain};
+```rust,no_run
+use bdk::{FeeRate, TxBuilder, Wallet};
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
 
-use magical::electrum_client::Client;
+use bdk::electrum_client::Client;
 
 use bitcoin::consensus::serialize;
 
-fn main() -> Result<(), magical::Error> {
-    let client = Client::new("ssl://electrum.blockstream.info:60002", None)?;
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
     let wallet = Wallet::new(
         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
@@ -122,13 +124,13 @@ fn main() -> Result<(), magical::Error> {
 
 ### Sign a transaction
 
-```no_run
-use magical::{Wallet, OfflineWallet};
-use magical::database::MemoryDatabase;
+```rust,no_run
+use bdk::{Wallet, OfflineWallet};
+use bdk::database::MemoryDatabase;
 
 use bitcoin::consensus::deserialize;
 
-fn main() -> Result<(), magical::Error> {
+fn main() -> Result<(), bdk::Error> {
     let wallet: OfflineWallet<_> = Wallet::new_offline(
         "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)",
         Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"),

ci/start-core.sh

@@ -1,24 +1,17 @@
 #!/usr/bin/env sh
 
-set -e
+echo "Starting bitcoin node."
+/root/bitcoind -regtest -server -daemon -fallbackfee=0.0002 -rpcuser=admin -rpcpassword=passw -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0
 
-BITCOIN_VERSION=0.20.1
-
-# This should be cached by Travis
-cargo install --git https://github.com/romanz/electrs --bin electrs
-
-curl -O -L https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz
-tar xf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz
-
-export PATH=$PATH:./bitcoin-$BITCOIN_VERSION/bin
-
-bitcoind -regtest=1 -daemon=1 -fallbackfee=0.0002
-until bitcoin-cli -regtest getblockchaininfo; do
+echo "Waiting for bitcoin node."
+until /root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw getblockchaininfo; do
     sleep 1
 done
 
-ADDR=$(bitcoin-cli -regtest getnewaddress)
-bitcoin-cli -regtest generatetoaddress 150 $ADDR
+echo "Generating 150 bitcoin blocks."
+ADDR=$(/root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw getnewaddress)
+/root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw generatetoaddress 150 $ADDR
 
-nohup electrs --network regtest --jsonrpc-import --cookie-file /home/travis/.bitcoin/regtest/.cookie &
+echo "Starting electrs node."
+nohup /root/electrs --network regtest --jsonrpc-import &
 sleep 5

examples/address_validator.rs

@@ -24,12 +24,12 @@
 
 use std::sync::Arc;
 
-use magical::bitcoin;
-use magical::database::MemoryDatabase;
-use magical::descriptor::HDKeyPaths;
-use magical::wallet::address_validator::{AddressValidator, AddressValidatorError};
-use magical::ScriptType;
-use magical::{OfflineWallet, Wallet};
+use bdk::bitcoin;
+use bdk::database::MemoryDatabase;
+use bdk::descriptor::HDKeyPaths;
+use bdk::wallet::address_validator::{AddressValidator, AddressValidatorError};
+use bdk::KeychainKind;
+use bdk::{OfflineWallet, Wallet};
 
 use bitcoin::hashes::hex::FromHex;
 use bitcoin::util::bip32::Fingerprint;
@@ -39,7 +39,7 @@ struct DummyValidator;
 impl AddressValidator for DummyValidator {
     fn validate(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         hd_keypaths: &HDKeyPaths,
         script: &Script,
     ) -> Result<(), AddressValidatorError> {
@@ -50,19 +50,19 @@ impl AddressValidator for DummyValidator {
 
         println!(
             "Validating `{:?}` {} address, script: {}",
-            script_type, path, script
+            keychain, path, script
         );
 
         Ok(())
     }
 }
 
-fn main() -> Result<(), magical::Error> {
+fn main() -> Result<(), bdk::Error> {
     let descriptor = "sh(and_v(v:pk(tpubDDpWvmUrPZrhSPmUzCMBHffvC3HyMAPnWDSAQNBTnj1iZeJa7BZQEttFiP4DS4GCcXQHezdXhn86Hj6LHX5EDstXPWrMaSneRWM8yUf6NFd/*),after(630000)))";
     let mut wallet: OfflineWallet<_> =
         Wallet::new_offline(descriptor, None, Network::Regtest, MemoryDatabase::new())?;
 
-    wallet.add_address_validator(Arc::new(Box::new(DummyValidator)));
+    wallet.add_address_validator(Arc::new(DummyValidator));
 
     wallet.get_new_address()?;
     wallet.get_new_address()?;

examples/compiler.rs

@@ -22,10 +22,10 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
+extern crate bdk;
 extern crate bitcoin;
 extern crate clap;
 extern crate log;
-extern crate magical;
 extern crate miniscript;
 extern crate serde_json;
 
@@ -39,8 +39,8 @@ use bitcoin::Network;
 use miniscript::policy::Concrete;
 use miniscript::Descriptor;
 
-use magical::database::memory::MemoryDatabase;
-use magical::{OfflineWallet, ScriptType, Wallet};
+use bdk::database::memory::MemoryDatabase;
+use bdk::{KeychainKind, OfflineWallet, Wallet};
 
 fn main() {
     env_logger::init_from_env(
@@ -104,7 +104,7 @@ fn main() {
     info!("... First address: {}", wallet.get_new_address().unwrap());
 
     if matches.is_present("parsed_policy") {
-        let spending_policy = wallet.policies(ScriptType::External).unwrap();
+        let spending_policy = wallet.policies(KeychainKind::External).unwrap();
         info!(
             "... Spending policy:\n{}",
             serde_json::to_string_pretty(&spending_policy).unwrap()

examples/parse_descriptor.rs

@@ -22,16 +22,20 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-extern crate magical;
+extern crate bdk;
 extern crate serde_json;
 
 use std::sync::Arc;
 
-use magical::bitcoin::util::bip32::ChildNumber;
-use magical::bitcoin::*;
-use magical::descriptor::*;
+use bdk::bitcoin::secp256k1::Secp256k1;
+use bdk::bitcoin::util::bip32::ChildNumber;
+use bdk::bitcoin::*;
+use bdk::descriptor::*;
+use bdk::miniscript::DescriptorPublicKeyCtx;
 
 fn main() {
+    let secp = Secp256k1::new();
+
     let desc = "wsh(or_d(\
                     multi(\
                       2,[d34db33f/44'/0'/0']xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*,tprv8ZgxMBicQKsPduL5QnGihpprdHyypMGi4DhimjtzYemu7se5YQNcZfAPLqXRuGHb5ZX2eTQj62oNqMnyxJ7B7wz54Uzswqw8fFqMVdcmVF7/1/*\
@@ -39,19 +43,18 @@ fn main() {
                     and_v(vc:pk_h(cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy),older(1000))\
                    ))";
 
-    let (extended_desc, key_map) = ExtendedDescriptor::parse_secret(desc).unwrap();
+    let (extended_desc, key_map) = ExtendedDescriptor::parse_descriptor(desc).unwrap();
     println!("{:?}", extended_desc);
 
+    let deriv_ctx = DescriptorPublicKeyCtx::new(&secp, ChildNumber::from_normal_idx(42).unwrap());
+
     let signers = Arc::new(key_map.into());
-    let policy = extended_desc.extract_policy(signers).unwrap();
+    let policy = extended_desc.extract_policy(&signers, &secp).unwrap();
     println!("policy: {}", serde_json::to_string(&policy).unwrap());
 
-    let derived_desc = extended_desc.derive(&[ChildNumber::from_normal_idx(42).unwrap()]);
-    println!("{:?}", derived_desc);
-
-    let addr = derived_desc.address(Network::Testnet).unwrap();
+    let addr = extended_desc.address(Network::Testnet, deriv_ctx).unwrap();
     println!("{}", addr);
 
-    let script = derived_desc.witness_script();
+    let script = extended_desc.witness_script(deriv_ctx);
     println!("{:?}", script);
 }

examples/repl.rs

@@ -24,28 +24,40 @@
 
 use std::fs;
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::sync::Arc;
 
+use bitcoin::Network;
+use clap::AppSettings;
+use log::{debug, info, warn, LevelFilter};
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
+use structopt::StructOpt;
 
-use clap::AppSettings;
+use bdk::bitcoin;
+#[cfg(feature = "esplora")]
+use bdk::blockchain::esplora::EsploraBlockchainConfig;
+use bdk::blockchain::{
+    AnyBlockchain, AnyBlockchainConfig, ConfigurableBlockchain, ElectrumBlockchainConfig,
+};
+use bdk::cli::{self, WalletOpt, WalletSubCommand};
+use bdk::sled;
+use bdk::Wallet;
 
-#[allow(unused_imports)]
-use log::{debug, error, info, trace, LevelFilter};
-
-use bitcoin::Network;
-
-use magical::bitcoin;
-use magical::blockchain::compact_filters::*;
-use magical::cli;
-use magical::sled;
-use magical::Wallet;
+#[derive(Debug, StructOpt, Clone, PartialEq)]
+#[structopt(name = "BDK Wallet", setting = AppSettings::NoBinaryName,
+version = option_env ! ("CARGO_PKG_VERSION").unwrap_or("unknown"),
+author = option_env ! ("CARGO_PKG_AUTHORS").unwrap_or(""))]
+struct ReplOpt {
+    /// Wallet sub-command
+    #[structopt(subcommand)]
+    pub subcommand: WalletSubCommand,
+}
 
 fn prepare_home_dir() -> PathBuf {
     let mut dir = PathBuf::new();
-    dir.push(&dirs::home_dir().unwrap());
-    dir.push(".magical-bitcoin");
+    dir.push(&dirs_next::home_dir().unwrap());
+    dir.push(".bdk-bitcoin");
 
     if !dir.exists() {
         info!("Creating home directory {}", dir.as_path().display());
@@ -57,88 +69,106 @@ fn prepare_home_dir() -> PathBuf {
 }
 
 fn main() {
-    env_logger::init();
+    let cli_opt: WalletOpt = WalletOpt::from_args();
 
-    let app = cli::make_cli_subcommands();
-    let mut repl_app = app.clone().setting(AppSettings::NoBinaryName);
+    let level = LevelFilter::from_str(cli_opt.log_level.as_str()).unwrap_or(LevelFilter::Info);
+    env_logger::builder().filter_level(level).init();
 
-    let app = cli::add_global_flags(app);
+    let network = Network::from_str(cli_opt.network.as_str()).unwrap_or(Network::Testnet);
+    debug!("network: {:?}", network);
+    if network == Network::Bitcoin {
+        warn!("This is experimental software and not currently recommended for use on Bitcoin mainnet, proceed with caution.")
+    }
 
-    let matches = app.get_matches();
-
-    // TODO
-    // let level = match matches.occurrences_of("v") {
-    //     0 => LevelFilter::Info,
-    //     1 => LevelFilter::Debug,
-    //     _ => LevelFilter::Trace,
-    // };
-
-    let network = match matches.value_of("network") {
-        Some("regtest") => Network::Regtest,
-        Some("testnet") | _ => Network::Testnet,
-    };
-
-    let descriptor = matches.value_of("descriptor").unwrap();
-    let change_descriptor = matches.value_of("change_descriptor");
+    let descriptor = cli_opt.descriptor.as_str();
+    let change_descriptor = cli_opt.change_descriptor.as_deref();
     debug!("descriptors: {:?} {:?}", descriptor, change_descriptor);
 
     let database = sled::open(prepare_home_dir().to_str().unwrap()).unwrap();
-    let tree = database
-        .open_tree(matches.value_of("wallet").unwrap())
-        .unwrap();
+    let tree = database.open_tree(cli_opt.wallet).unwrap();
     debug!("database opened successfully");
 
-    let num_threads = 1;
+    // Try to use Esplora config if "esplora" feature is enabled
+    #[cfg(feature = "esplora")]
+    let config_esplora: Option<AnyBlockchainConfig> = {
+        let esplora_concurrency = cli_opt.esplora_concurrency;
+        cli_opt.esplora.map(|base_url| {
+            AnyBlockchainConfig::Esplora(EsploraBlockchainConfig {
+                base_url: base_url.to_string(),
+                concurrency: Some(esplora_concurrency),
+            })
+        })
+    };
+    #[cfg(not(feature = "esplora"))]
+    let config_esplora = None;
 
-    let mempool = Arc::new(Mempool::default());
-    let peers = (0..num_threads)
-        .map(|_| Peer::connect("192.168.1.136:8333", Arc::clone(&mempool), Network::Bitcoin))
-        .collect::<Result<_, _>>()
-        .unwrap();
-    let blockchain =
-        CompactFiltersBlockchain::new(peers, "./wallet-filters", Some(500_000)).unwrap();
+    // Fall back to Electrum config if Esplora config isn't provided
+    let config =
+        config_esplora.unwrap_or(AnyBlockchainConfig::Electrum(ElectrumBlockchainConfig {
+            url: cli_opt.electrum,
+            socks5: cli_opt.proxy,
+            retry: 10,
+            timeout: 10,
+        }));
+
+    let wallet = Wallet::new(
+        descriptor,
+        change_descriptor,
+        network,
+        tree,
+        AnyBlockchain::from_config(&config).unwrap(),
+    )
+    .unwrap();
 
-    let wallet = Wallet::new(descriptor, change_descriptor, network, tree, blockchain).unwrap();
     let wallet = Arc::new(wallet);
 
-    if let Some(_sub_matches) = matches.subcommand_matches("repl") {
-        let mut rl = Editor::<()>::new();
+    match cli_opt.subcommand {
+        WalletSubCommand::Other(external) if external.contains(&"repl".to_string()) => {
+            let mut rl = Editor::<()>::new();
 
-        // if rl.load_history("history.txt").is_err() {
-        //     println!("No previous history.");
-        // }
+            // if rl.load_history("history.txt").is_err() {
+            //     println!("No previous history.");
+            // }
 
-        loop {
-            let readline = rl.readline(">> ");
-            match readline {
-                Ok(line) => {
-                    if line.trim() == "" {
-                        continue;
+            loop {
+                let readline = rl.readline(">> ");
+                match readline {
+                    Ok(line) => {
+                        if line.trim() == "" {
+                            continue;
+                        }
+                        rl.add_history_entry(line.as_str());
+                        let split_line: Vec<&str> = line.split(" ").collect();
+                        let repl_subcommand: Result<ReplOpt, clap::Error> =
+                            ReplOpt::from_iter_safe(split_line);
+                        debug!("repl_subcommand = {:?}", repl_subcommand);
+
+                        if let Err(err) = repl_subcommand {
+                            println!("{}", err.message);
+                            continue;
+                        }
+
+                        let result = cli::handle_wallet_subcommand(
+                            &Arc::clone(&wallet),
+                            repl_subcommand.unwrap().subcommand,
+                        )
+                        .unwrap();
+                        println!("{}", serde_json::to_string_pretty(&result).unwrap());
                     }
-
-                    rl.add_history_entry(line.as_str());
-                    let matches = repl_app.get_matches_from_safe_borrow(line.split(" "));
-                    if let Err(err) = matches {
-                        println!("{}", err.message);
-                        continue;
+                    Err(ReadlineError::Interrupted) => continue,
+                    Err(ReadlineError::Eof) => break,
+                    Err(err) => {
+                        println!("{:?}", err);
+                        break;
                     }
-
-                    let result =
-                        cli::handle_matches(&Arc::clone(&wallet), matches.unwrap()).unwrap();
-                    println!("{}", serde_json::to_string_pretty(&result).unwrap());
-                }
-                Err(ReadlineError::Interrupted) => continue,
-                Err(ReadlineError::Eof) => break,
-                Err(err) => {
-                    println!("{:?}", err);
-                    break;
                 }
             }
-        }
 
-    // rl.save_history("history.txt").unwrap();
-    } else {
-        let result = cli::handle_matches(&wallet, matches).unwrap();
-        println!("{}", serde_json::to_string_pretty(&result).unwrap());
+            // rl.save_history("history.txt").unwrap();
+        }
+        _ => {
+            let result = cli::handle_wallet_subcommand(&wallet, cli_opt.subcommand).unwrap();
+            println!("{}", serde_json::to_string_pretty(&result).unwrap());
+        }
     }
 }

macros/Cargo.toml

@@ -1,8 +1,14 @@
 [package]
-name = "magical-macros"
-version = "0.1.0-beta.1"
+name = "bdk-macros"
+version = "0.2.0"
 authors = ["Alekos Filini <alekos.filini@gmail.com>"]
 edition = "2018"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk-macros"
+description = "Supporting macros for `bdk`"
+keywords = ["bdk"]
+license-file = "../LICENSE"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

src/blockchain/any.rs

@@ -0,0 +1,232 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Runtime-checked blockchain types
+//!
+//! This module provides the implementation of [`AnyBlockchain`] which allows switching the
+//! inner [`Blockchain`] type at runtime.
+//!
+//! ## Example
+//!
+//! In this example both `wallet_electrum` and `wallet_esplora` have the same type of
+//! `Wallet<AnyBlockchain, MemoryDatabase>`. This means that they could both, for instance, be
+//! assigned to a struct member.
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::blockchain::*;
+//! # use bdk::database::MemoryDatabase;
+//! # use bdk::Wallet;
+//! # #[cfg(feature = "electrum")]
+//! # {
+//! let electrum_blockchain = ElectrumBlockchain::from(electrum_client::Client::new("...")?);
+//! let wallet_electrum: Wallet<AnyBlockchain, _> = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     electrum_blockchain.into(),
+//! )?;
+//! # }
+//!
+//! # #[cfg(feature = "esplora")]
+//! # {
+//! let esplora_blockchain = EsploraBlockchain::new("...", None);
+//! let wallet_esplora: Wallet<AnyBlockchain, _> = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     esplora_blockchain.into(),
+//! )?;
+//! # }
+//!
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! When paired with the use of [`ConfigurableBlockchain`], it allows creating wallets with any
+//! blockchain type supported using a single line of code:
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::blockchain::*;
+//! # use bdk::database::MemoryDatabase;
+//! # use bdk::Wallet;
+//! let config = serde_json::from_str("...")?;
+//! let blockchain = AnyBlockchain::from_config(&config)?;
+//! let wallet = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     blockchain,
+//! )?;
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use super::*;
+
+macro_rules! impl_from {
+    ( $from:ty, $to:ty, $variant:ident, $( $cfg:tt )* ) => {
+        $( $cfg )*
+        impl From<$from> for $to {
+            fn from(inner: $from) -> Self {
+                <$to>::$variant(inner)
+            }
+        }
+    };
+}
+
+macro_rules! impl_inner_method {
+    ( $self:expr, $name:ident $(, $args:expr)* ) => {
+        match $self {
+            #[cfg(feature = "electrum")]
+            AnyBlockchain::Electrum(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "esplora")]
+            AnyBlockchain::Esplora(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "compact_filters")]
+            AnyBlockchain::CompactFilters(inner) => inner.$name( $($args, )* ),
+        }
+    }
+}
+
+/// Type that can contain any of the [`Blockchain`] types defined by the library
+///
+/// It allows switching backend at runtime
+///
+/// See [this module](crate::blockchain::any)'s documentation for a usage example.
+pub enum AnyBlockchain {
+    #[cfg(feature = "electrum")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
+    /// Electrum client
+    Electrum(electrum::ElectrumBlockchain),
+    #[cfg(feature = "esplora")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
+    /// Esplora client
+    Esplora(esplora::EsploraBlockchain),
+    #[cfg(feature = "compact_filters")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
+    /// Compact filters client
+    CompactFilters(compact_filters::CompactFiltersBlockchain),
+}
+
+#[maybe_async]
+impl Blockchain for AnyBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        maybe_await!(impl_inner_method!(self, get_capabilities))
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(
+            self,
+            setup,
+            stop_gap,
+            database,
+            progress_update
+        ))
+    }
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(
+            self,
+            sync,
+            stop_gap,
+            database,
+            progress_update
+        ))
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        maybe_await!(impl_inner_method!(self, get_tx, txid))
+    }
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(self, broadcast, tx))
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        maybe_await!(impl_inner_method!(self, get_height))
+    }
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        maybe_await!(impl_inner_method!(self, estimate_fee, target))
+    }
+}
+
+impl_from!(electrum::ElectrumBlockchain, AnyBlockchain, Electrum, #[cfg(feature = "electrum")]);
+impl_from!(esplora::EsploraBlockchain, AnyBlockchain, Esplora, #[cfg(feature = "esplora")]);
+impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilters, #[cfg(feature = "compact_filters")]);
+
+/// Type that can contain any of the blockchain configurations defined by the library
+///
+/// This allows storing a single configuration that can be loaded into an [`AnyBlockchain`]
+/// instance. Wallets that plan to offer users the ability to switch blockchain backend at runtime
+/// will find this particularly useful.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub enum AnyBlockchainConfig {
+    #[cfg(feature = "electrum")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
+    /// Electrum client
+    Electrum(electrum::ElectrumBlockchainConfig),
+    #[cfg(feature = "esplora")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
+    /// Esplora client
+    Esplora(esplora::EsploraBlockchainConfig),
+    #[cfg(feature = "compact_filters")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
+    /// Compact filters client
+    CompactFilters(compact_filters::CompactFiltersBlockchainConfig),
+}
+
+impl ConfigurableBlockchain for AnyBlockchain {
+    type Config = AnyBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(match config {
+            #[cfg(feature = "electrum")]
+            AnyBlockchainConfig::Electrum(inner) => {
+                AnyBlockchain::Electrum(electrum::ElectrumBlockchain::from_config(inner)?)
+            }
+            #[cfg(feature = "esplora")]
+            AnyBlockchainConfig::Esplora(inner) => {
+                AnyBlockchain::Esplora(esplora::EsploraBlockchain::from_config(inner)?)
+            }
+            #[cfg(feature = "compact_filters")]
+            AnyBlockchainConfig::CompactFilters(inner) => AnyBlockchain::CompactFilters(
+                compact_filters::CompactFiltersBlockchain::from_config(inner)?,
+            ),
+        })
+    }
+}
+
+impl_from!(electrum::ElectrumBlockchainConfig, AnyBlockchainConfig, Electrum, #[cfg(feature = "electrum")]);
+impl_from!(esplora::EsploraBlockchainConfig, AnyBlockchainConfig, Esplora, #[cfg(feature = "esplora")]);
+impl_from!(compact_filters::CompactFiltersBlockchainConfig, AnyBlockchainConfig, CompactFilters, #[cfg(feature = "compact_filters")]);

src/blockchain/compact_filters/mod.rs

@@ -24,7 +24,7 @@
 
 //! Compact Filters
 //!
-//! This module contains a multithreaded implementation of an [`OnlineBlockchain`] backend that
+//! This module contains a multithreaded implementation of an [`Blockchain`] backend that
 //! uses BIP157 (aka "Neutrino") to populate the wallet's [database](crate::database::Database)
 //! by downloading compact filters from the P2P network.
 //!
@@ -37,25 +37,29 @@
 //! connecting to a single peer at a time, optionally by opening multiple connections if it's
 //! desirable to use multiple threads at once to sync in parallel.
 //!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
 //! ## Example
 //!
 //! ```no_run
 //! # use std::sync::Arc;
 //! # use bitcoin::*;
-//! # use magical::*;
-//! # use magical::blockchain::compact_filters::*;
+//! # use bdk::*;
+//! # use bdk::blockchain::compact_filters::*;
 //! let num_threads = 4;
 //!
 //! let mempool = Arc::new(Mempool::default());
 //! let peers = (0..num_threads)
-//!     .map(|_| Peer::connect(
-//!         "btcd-mainnet.lightning.computer:8333",
-//!         Arc::clone(&mempool),
-//!         Network::Bitcoin,
-//!     ))
+//!     .map(|_| {
+//!         Peer::connect(
+//!             "btcd-mainnet.lightning.computer:8333",
+//!             Arc::clone(&mempool),
+//!             Network::Bitcoin,
+//!         )
+//!     })
 //!     .collect::<Result<_, _>>()?;
 //! let blockchain = CompactFiltersBlockchain::new(peers, "./wallet-filters", Some(500_000))?;
-//! # Ok::<(), magical::error::Error>(())
+//! # Ok::<(), CompactFiltersError>(())
 //! ```
 
 use std::collections::HashSet;
@@ -68,7 +72,7 @@ use std::sync::{Arc, Mutex};
 use log::{debug, error, info, trace};
 
 use bitcoin::network::message_blockdata::Inventory;
-use bitcoin::{OutPoint, Transaction, Txid};
+use bitcoin::{Network, OutPoint, Transaction, Txid};
 
 use rocksdb::{Options, SliceTransform, DB};
 
@@ -76,10 +80,10 @@ mod peer;
 mod store;
 mod sync;
 
-use super::{Blockchain, Capability, OnlineBlockchain, Progress};
+use super::{Blockchain, Capability, ConfigurableBlockchain, Progress};
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{ScriptType, TransactionDetails, UTXO};
+use crate::types::{KeychainKind, TransactionDetails, UTXO};
 use crate::FeeRate;
 
 use peer::*;
@@ -97,7 +101,11 @@ const PROCESS_BLOCKS_COST: f32 = 20_000.0;
 /// ## Example
 /// See the [`blockchain::compact_filters`](crate::blockchain::compact_filters) module for a usage example.
 #[derive(Debug)]
-pub struct CompactFiltersBlockchain(Option<CompactFilters>);
+pub struct CompactFiltersBlockchain {
+    peers: Vec<Arc<Peer>>,
+    headers: Arc<ChainStore<Full>>,
+    skip_blocks: Option<usize>,
+}
 
 impl CompactFiltersBlockchain {
     /// Construct a new instance given a list of peers, a path to store headers and block
@@ -108,29 +116,6 @@ impl CompactFiltersBlockchain {
     /// in parallel. It's currently recommended to only connect to a single peer to avoid
     /// inconsistencies in the data returned, optionally with multiple connections in parallel to
     /// speed-up the sync process.
-    pub fn new<P: AsRef<Path>>(
-        peers: Vec<Peer>,
-        storage_dir: P,
-        skip_blocks: Option<usize>,
-    ) -> Result<Self, CompactFiltersError> {
-        Ok(CompactFiltersBlockchain(Some(CompactFilters::new(
-            peers,
-            storage_dir,
-            skip_blocks,
-        )?)))
-    }
-}
-
-/// Internal struct that contains the state of a [`CompactFiltersBlockchain`]
-#[derive(Debug)]
-struct CompactFilters {
-    peers: Vec<Arc<Peer>>,
-    headers: Arc<ChainStore<Full>>,
-    skip_blocks: Option<usize>,
-}
-
-impl CompactFilters {
-    /// Constructor, see [`CompactFiltersBlockchain::new`] for the documentation
     pub fn new<P: AsRef<Path>>(
         peers: Vec<Peer>,
         storage_dir: P,
@@ -160,7 +145,7 @@ impl CompactFilters {
             headers.recover_snapshot(cf_name)?;
         }
 
-        Ok(CompactFilters {
+        Ok(CompactFiltersBlockchain {
             peers: peers.into_iter().map(Arc::new).collect(),
             headers,
             skip_blocks,
@@ -205,22 +190,22 @@ impl CompactFilters {
             outputs_sum += output.value;
 
             // this output is ours, we have a path to derive it
-            if let Some((script_type, child)) =
+            if let Some((keychain, child)) =
                 database.get_path_from_script_pubkey(&output.script_pubkey)?
             {
                 debug!("{} output #{} is mine, adding utxo", tx.txid(), i);
                 updates.set_utxo(&UTXO {
                     outpoint: OutPoint::new(tx.txid(), i as u32),
                     txout: output.clone(),
-                    is_internal: script_type.is_internal(),
+                    keychain,
                 })?;
                 incoming += output.value;
 
-                if script_type == ScriptType::Internal
+                if keychain == KeychainKind::Internal
                     && (internal_max_deriv.is_none() || child > internal_max_deriv.unwrap_or(0))
                 {
                     *internal_max_deriv = Some(child);
-                } else if script_type == ScriptType::External
+                } else if keychain == KeychainKind::External
                     && (external_max_deriv.is_none() || child > external_max_deriv.unwrap_or(0))
                 {
                     *external_max_deriv = Some(child);
@@ -250,16 +235,6 @@ impl CompactFilters {
 }
 
 impl Blockchain for CompactFiltersBlockchain {
-    fn offline() -> Self {
-        CompactFiltersBlockchain(None)
-    }
-
-    fn is_online(&self) -> bool {
-        self.0.is_some()
-    }
-}
-
-impl OnlineBlockchain for CompactFiltersBlockchain {
     fn get_capabilities(&self) -> HashSet<Capability> {
         vec![Capability::FullHistory].into_iter().collect()
     }
@@ -270,14 +245,13 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-        let first_peer = &inner.peers[0];
+        let first_peer = &self.peers[0];
 
-        let skip_blocks = inner.skip_blocks.unwrap_or(0);
+        let skip_blocks = self.skip_blocks.unwrap_or(0);
 
-        let cf_sync = Arc::new(CFSync::new(Arc::clone(&inner.headers), skip_blocks, 0x00)?);
+        let cf_sync = Arc::new(CFSync::new(Arc::clone(&self.headers), skip_blocks, 0x00)?);
 
-        let initial_height = inner.headers.get_height()?;
+        let initial_height = self.headers.get_height()?;
         let total_bundles = (first_peer.get_version().start_height as usize)
             .checked_sub(skip_blocks)
             .map(|x| x / 1000)
@@ -297,7 +271,7 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
 
         if let Some(snapshot) = sync::sync_headers(
             Arc::clone(&first_peer),
-            Arc::clone(&inner.headers),
+            Arc::clone(&self.headers),
             |new_height| {
                 let local_headers_cost =
                     new_height.checked_sub(initial_height).unwrap_or(0) as f32 * SYNC_HEADERS_COST;
@@ -307,13 +281,13 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
                 )
             },
         )? {
-            if snapshot.work()? > inner.headers.work()? {
+            if snapshot.work()? > self.headers.work()? {
                 info!("Applying snapshot with work: {}", snapshot.work()?);
-                inner.headers.apply_snapshot(snapshot)?;
+                self.headers.apply_snapshot(snapshot)?;
             }
         }
 
-        let synced_height = inner.headers.get_height()?;
+        let synced_height = self.headers.get_height()?;
         let buried_height = synced_height
             .checked_sub(sync::BURIED_CONFIRMATIONS)
             .unwrap_or(0);
@@ -333,11 +307,11 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
         let synced_bundles = Arc::new(AtomicUsize::new(0));
         let progress_update = Arc::new(Mutex::new(progress_update));
 
-        let mut threads = Vec::with_capacity(inner.peers.len());
-        for peer in &inner.peers {
+        let mut threads = Vec::with_capacity(self.peers.len());
+        for peer in &self.peers {
             let cf_sync = Arc::clone(&cf_sync);
             let peer = Arc::clone(&peer);
-            let headers = Arc::clone(&inner.headers);
+            let headers = Arc::clone(&self.headers);
             let all_scripts = Arc::clone(&all_scripts);
             let last_synced_block = Arc::clone(&last_synced_block);
             let progress_update = Arc::clone(&progress_update);
@@ -420,9 +394,9 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
         let mut internal_max_deriv = None;
         let mut external_max_deriv = None;
 
-        for (height, block) in inner.headers.iter_full_blocks()? {
+        for (height, block) in self.headers.iter_full_blocks()? {
             for tx in &block.txdata {
-                inner.process_tx(
+                self.process_tx(
                     database,
                     tx,
                     Some(height as u32),
@@ -433,7 +407,7 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
             }
         }
         for tx in first_peer.get_mempool().iter_txs().iter() {
-            inner.process_tx(
+            self.process_tx(
                 database,
                 tx,
                 None,
@@ -443,22 +417,26 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
             )?;
         }
 
-        let current_ext = database.get_last_index(ScriptType::External)?.unwrap_or(0);
+        let current_ext = database
+            .get_last_index(KeychainKind::External)?
+            .unwrap_or(0);
         let first_ext_new = external_max_deriv.map(|x| x + 1).unwrap_or(0);
         if first_ext_new > current_ext {
             info!("Setting external index to {}", first_ext_new);
-            database.set_last_index(ScriptType::External, first_ext_new)?;
+            database.set_last_index(KeychainKind::External, first_ext_new)?;
         }
 
-        let current_int = database.get_last_index(ScriptType::Internal)?.unwrap_or(0);
+        let current_int = database
+            .get_last_index(KeychainKind::Internal)?
+            .unwrap_or(0);
         let first_int_new = internal_max_deriv.map(|x| x + 1).unwrap_or(0);
         if first_int_new > current_int {
             info!("Setting internal index to {}", first_int_new);
-            database.set_last_index(ScriptType::Internal, first_int_new)?;
+            database.set_last_index(KeychainKind::Internal, first_int_new)?;
         }
 
         info!("Dropping blocks until {}", buried_height);
-        inner.headers.delete_blocks_until(buried_height)?;
+        self.headers.delete_blocks_until(buried_height)?;
 
         progress_update
             .lock()
@@ -469,24 +447,19 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-
-        Ok(inner.peers[0]
+        Ok(self.peers[0]
             .get_mempool()
             .get_tx(&Inventory::Transaction(*txid)))
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-        inner.peers[0].broadcast_tx(tx.clone())?;
+        self.peers[0].broadcast_tx(tx.clone())?;
 
         Ok(())
     }
 
     fn get_height(&self) -> Result<u32, Error> {
-        let inner = self.0.as_ref().ok_or(Error::OfflineClient)?;
-
-        Ok(inner.headers.get_height()? as u32)
+        Ok(self.headers.get_height()? as u32)
     }
 
     fn estimate_fee(&self, _target: usize) -> Result<FeeRate, Error> {
@@ -495,6 +468,61 @@ impl OnlineBlockchain for CompactFiltersBlockchain {
     }
 }
 
+/// Data to connect to a Bitcoin P2P peer
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct BitcoinPeerConfig {
+    /// Peer address such as 127.0.0.1:18333
+    pub address: String,
+    /// Optional socks5 proxy
+    pub socks5: Option<String>,
+    /// Optional socks5 proxy credentials
+    pub socks5_credentials: Option<(String, String)>,
+}
+
+/// Configuration for a [`CompactFiltersBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct CompactFiltersBlockchainConfig {
+    /// List of peers to try to connect to for asking headers and filters
+    pub peers: Vec<BitcoinPeerConfig>,
+    /// Network used
+    pub network: Network,
+    /// Storage dir to save partially downloaded headers and full blocks
+    pub storage_dir: String,
+    /// Optionally skip initial `skip_blocks` blocks (default: 0)
+    pub skip_blocks: Option<usize>,
+}
+
+impl ConfigurableBlockchain for CompactFiltersBlockchain {
+    type Config = CompactFiltersBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let mempool = Arc::new(Mempool::default());
+        let peers = config
+            .peers
+            .iter()
+            .map(|peer_conf| match &peer_conf.socks5 {
+                None => Peer::connect(&peer_conf.address, Arc::clone(&mempool), config.network),
+                Some(proxy) => Peer::connect_proxy(
+                    peer_conf.address.as_str(),
+                    proxy,
+                    peer_conf
+                        .socks5_credentials
+                        .as_ref()
+                        .map(|(a, b)| (a.as_str(), b.as_str())),
+                    Arc::clone(&mempool),
+                    config.network,
+                ),
+            })
+            .collect::<Result<_, _>>()?;
+
+        Ok(CompactFiltersBlockchain::new(
+            peers,
+            &config.storage_dir,
+            config.skip_blocks,
+        )?)
+    }
+}
+
 /// An error that can occur during sync with a [`CompactFiltersBlockchain`]
 #[derive(Debug)]
 pub enum CompactFiltersError {
@@ -540,20 +568,10 @@ impl fmt::Display for CompactFiltersError {
 
 impl std::error::Error for CompactFiltersError {}
 
-macro_rules! impl_error {
-    ( $from:ty, $to:ident ) => {
-        impl std::convert::From<$from> for CompactFiltersError {
-            fn from(err: $from) -> Self {
-                CompactFiltersError::$to(err)
-            }
-        }
-    };
-}
-
-impl_error!(rocksdb::Error, DB);
-impl_error!(std::io::Error, IO);
-impl_error!(bitcoin::util::bip158::Error, BIP158);
-impl_error!(std::time::SystemTimeError, Time);
+impl_error!(rocksdb::Error, DB, CompactFiltersError);
+impl_error!(std::io::Error, IO, CompactFiltersError);
+impl_error!(bitcoin::util::bip158::Error, BIP158, CompactFiltersError);
+impl_error!(std::time::SystemTimeError, Time, CompactFiltersError);
 
 impl From<crate::error::Error> for CompactFiltersError {
     fn from(err: crate::error::Error) -> Self {

src/blockchain/compact_filters/peer.rs

@@ -63,7 +63,7 @@ impl Mempool {
     /// Add a transaction to the mempool
     ///
     /// Note that this doesn't propagate the transaction to other
-    /// peers. To do that, [`broadcast`](crate::blockchain::OnlineBlockchain::broadcast) should be used.
+    /// peers. To do that, [`broadcast`](crate::blockchain::Blockchain::broadcast) should be used.
     pub fn add_tx(&self, tx: Transaction) {
         self.txs.write().unwrap().insert(tx.txid(), tx);
     }
@@ -108,7 +108,7 @@ impl Peer {
     /// Connect to a peer over a plaintext TCP connection
     ///
     /// This function internally spawns a new thread that will monitor incoming messages from the
-    /// peer, and optionally reply to some of them transparently, like [pings](NetworkMessage::Ping)
+    /// peer, and optionally reply to some of them transparently, like [pings](bitcoin::network::message::NetworkMessage::Ping)
     pub fn connect<A: ToSocketAddrs>(
         address: A,
         mempool: Arc<Mempool>,

src/blockchain/electrum.rs

@@ -24,17 +24,17 @@
 
 //! Electrum
 //!
-//! This module defines an [`OnlineBlockchain`] struct that wraps an [`electrum_client::Client`]
+//! This module defines a [`Blockchain`] struct that wraps an [`electrum_client::Client`]
 //! and implements the logic required to populate the wallet's [database](crate::database::Database) by
 //! querying the inner client.
 //!
 //! ## Example
 //!
 //! ```no_run
-//! # use magical::blockchain::electrum::ElectrumBlockchain;
-//! let client = electrum_client::Client::new("ssl://electrum.blockstream.info:50002", None)?;
+//! # use bdk::blockchain::electrum::ElectrumBlockchain;
+//! let client = electrum_client::Client::new("ssl://electrum.blockstream.info:50002")?;
 //! let blockchain = ElectrumBlockchain::from(client);
-//! # Ok::<(), magical::Error>(())
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::collections::HashSet;
@@ -42,11 +42,11 @@ use std::collections::HashSet;
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
 
-use electrum_client::{Client, ElectrumApi};
+use electrum_client::{Client, ConfigBuilder, ElectrumApi, Socks5Config};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
 use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
@@ -56,32 +56,22 @@ use crate::FeeRate;
 ///
 /// ## Example
 /// See the [`blockchain::electrum`](crate::blockchain::electrum) module for a usage example.
-pub struct ElectrumBlockchain(Option<Client>);
+pub struct ElectrumBlockchain(Client);
 
 #[cfg(test)]
 #[cfg(feature = "test-electrum")]
-#[magical_blockchain_tests(crate)]
+#[bdk_blockchain_tests(crate)]
 fn local_electrs() -> ElectrumBlockchain {
-    ElectrumBlockchain::from(Client::new(&testutils::get_electrum_url(), None).unwrap())
+    ElectrumBlockchain::from(Client::new(&testutils::get_electrum_url()).unwrap())
 }
 
 impl std::convert::From<Client> for ElectrumBlockchain {
     fn from(client: Client) -> Self {
-        ElectrumBlockchain(Some(client))
+        ElectrumBlockchain(client)
     }
 }
 
 impl Blockchain for ElectrumBlockchain {
-    fn offline() -> Self {
-        ElectrumBlockchain(None)
-    }
-
-    fn is_online(&self) -> bool {
-        self.0.is_some()
-    }
-}
-
-impl OnlineBlockchain for ElectrumBlockchain {
     fn get_capabilities(&self) -> HashSet<Capability> {
         vec![
             Capability::FullHistory,
@@ -99,27 +89,15 @@ impl OnlineBlockchain for ElectrumBlockchain {
         progress_update: P,
     ) -> Result<(), Error> {
         self.0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
             .electrum_like_setup(stop_gap, database, progress_update)
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            .transaction_get(txid)
-            .map(Option::Some)?)
+        Ok(self.0.transaction_get(txid).map(Option::Some)?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            .transaction_broadcast(tx)
-            .map(|_| ())?)
+        Ok(self.0.transaction_broadcast(tx).map(|_| ())?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
@@ -127,24 +105,19 @@ impl OnlineBlockchain for ElectrumBlockchain {
 
         Ok(self
             .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
             .block_headers_subscribe()
             .map(|data| data.height as u32)?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
         Ok(FeeRate::from_btc_per_kvb(
-            self.0
-                .as_ref()
-                .ok_or(Error::OfflineClient)?
-                .estimate_fee(target)? as f32,
+            self.0.estimate_fee(target)? as f32
         ))
     }
 }
 
 impl ElectrumLikeSync for Client {
-    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
         &self,
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
@@ -168,35 +141,50 @@ impl ElectrumLikeSync for Client {
             .map_err(Error::Electrum)
     }
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid> + Clone>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
-        self.batch_script_list_unspent(scripts)
-            .map(|v| {
-                v.into_iter()
-                    .map(|v| {
-                        v.into_iter()
-                            .map(
-                                |electrum_client::ListUnspentRes {
-                                     height,
-                                     tx_hash,
-                                     tx_pos,
-                                     ..
-                                 }| ELSListUnspentRes {
-                                    height,
-                                    tx_hash,
-                                    tx_pos,
-                                },
-                            )
-                            .collect()
-                    })
-                    .collect()
-            })
-            .map_err(Error::Electrum)
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
+        self.batch_transaction_get(txids).map_err(Error::Electrum)
     }
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error> {
-        self.transaction_get(txid).map_err(Error::Electrum)
+    fn els_batch_block_header<I: IntoIterator<Item = u32> + Clone>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        self.batch_block_header(heights).map_err(Error::Electrum)
+    }
+}
+
+/// Configuration for an [`ElectrumBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct ElectrumBlockchainConfig {
+    /// URL of the Electrum server (such as ElectrumX, Esplora, BWT) may start with `ssl://` or `tcp://` and include a port
+    ///
+    /// eg. `ssl://electrum.blockstream.info:60002`
+    pub url: String,
+    /// URL of the socks5 proxy server or a Tor service
+    pub socks5: Option<String>,
+    /// Request retry count
+    pub retry: u8,
+    /// Request timeout (seconds)
+    pub timeout: u8,
+}
+
+impl ConfigurableBlockchain for ElectrumBlockchain {
+    type Config = ElectrumBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let socks5 = config.socks5.as_ref().map(Socks5Config::new);
+        let electrum_config = ConfigBuilder::new()
+            .retry(config.retry)
+            .socks5(socks5)?
+            .timeout(config.timeout)?
+            .build();
+
+        Ok(ElectrumBlockchain(Client::from_config(
+            config.url.as_str(),
+            electrum_config,
+        )?))
     }
 }

src/blockchain/esplora.rs

@@ -24,21 +24,21 @@
 
 //! Esplora
 //!
-//! This module defines an [`OnlineBlockchain`] struct that can query an Esplora backend
+//! This module defines a [`Blockchain`] struct that can query an Esplora backend
 //! populate the wallet's [database](crate::database::Database) by
 //!
 //! ## Example
 //!
 //! ```no_run
-//! # use magical::blockchain::esplora::EsploraBlockchain;
-//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api");
-//! # Ok::<(), magical::Error>(())
+//! # use bdk::blockchain::esplora::EsploraBlockchain;
+//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", None);
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::collections::{HashMap, HashSet};
 use std::fmt;
 
-use futures::stream::{self, StreamExt, TryStreamExt};
+use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
@@ -47,23 +47,27 @@ use serde::Deserialize;
 
 use reqwest::{Client, StatusCode};
 
-use bitcoin::consensus::{deserialize, serialize};
-use bitcoin::hashes::hex::ToHex;
+use bitcoin::consensus::{self, deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
 use bitcoin::hashes::{sha256, Hash};
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHash, BlockHeader, Script, Transaction, Txid};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
 use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
+use crate::wallet::utils::ChunksIterator;
 use crate::FeeRate;
 
+const DEFAULT_CONCURRENT_REQUESTS: u8 = 4;
+
 #[derive(Debug)]
 struct UrlClient {
     url: String,
     // We use the async client instead of the blocking one because it automatically uses `fetch`
     // when the target platform is wasm32.
     client: Client,
+    concurrency: u8,
 }
 
 /// Structure that implements the logic to sync with Esplora
@@ -71,36 +75,27 @@ struct UrlClient {
 /// ## Example
 /// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
 #[derive(Debug)]
-pub struct EsploraBlockchain(Option<UrlClient>);
+pub struct EsploraBlockchain(UrlClient);
 
 impl std::convert::From<UrlClient> for EsploraBlockchain {
     fn from(url_client: UrlClient) -> Self {
-        EsploraBlockchain(Some(url_client))
+        EsploraBlockchain(url_client)
     }
 }
 
 impl EsploraBlockchain {
     /// Create a new instance of the client from a base URL
-    pub fn new(base_url: &str) -> Self {
-        EsploraBlockchain(Some(UrlClient {
+    pub fn new(base_url: &str, concurrency: Option<u8>) -> Self {
+        EsploraBlockchain(UrlClient {
             url: base_url.to_string(),
             client: Client::new(),
-        }))
-    }
-}
-
-impl Blockchain for EsploraBlockchain {
-    fn offline() -> Self {
-        EsploraBlockchain(None)
-    }
-
-    fn is_online(&self) -> bool {
-        self.0.is_some()
+            concurrency: concurrency.unwrap_or(DEFAULT_CONCURRENT_REQUESTS),
+        })
     }
 }
 
 #[maybe_async]
-impl OnlineBlockchain for EsploraBlockchain {
+impl Blockchain for EsploraBlockchain {
     fn get_capabilities(&self) -> HashSet<Capability> {
         vec![
             Capability::FullHistory,
@@ -119,41 +114,23 @@ impl OnlineBlockchain for EsploraBlockchain {
     ) -> Result<(), Error> {
         maybe_await!(self
             .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
             .electrum_like_setup(stop_gap, database, progress_update))
     }
 
     fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._get_tx(txid))?)
+        Ok(await_or_block!(self.0._get_tx(txid))?)
     }
 
     fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
-        Ok(await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._broadcast(tx))?)
+        Ok(await_or_block!(self.0._broadcast(tx))?)
     }
 
     fn get_height(&self) -> Result<u32, Error> {
-        Ok(await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._get_height())?)
+        Ok(await_or_block!(self.0._get_height())?)
     }
 
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
-        let estimates = await_or_block!(self
-            .0
-            .as_ref()
-            .ok_or(Error::OfflineClient)?
-            ._get_fee_estimates())?;
+        let estimates = await_or_block!(self.0._get_fee_estimates())?;
 
         let fee_val = estimates
             .into_iter()
@@ -189,6 +166,39 @@ impl UrlClient {
         Ok(Some(deserialize(&resp.error_for_status()?.bytes().await?)?))
     }
 
+    async fn _get_tx_no_opt(&self, txid: &Txid) -> Result<Transaction, EsploraError> {
+        match self._get_tx(txid).await {
+            Ok(Some(tx)) => Ok(tx),
+            Ok(None) => Err(EsploraError::TransactionNotFound(*txid)),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn _get_header(&self, block_height: u32) -> Result<BlockHeader, EsploraError> {
+        let resp = self
+            .client
+            .get(&format!("{}/block-height/{}", self.url, block_height))
+            .send()
+            .await?;
+
+        if let StatusCode::NOT_FOUND = resp.status() {
+            return Err(EsploraError::HeaderHeightNotFound(block_height));
+        }
+        let bytes = resp.bytes().await?;
+        let hash = std::str::from_utf8(&bytes)
+            .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?;
+
+        let resp = self
+            .client
+            .get(&format!("{}/block/{}/header", self.url, hash))
+            .send()
+            .await?;
+
+        let header = deserialize(&Vec::from_hex(&resp.text().await?)?)?;
+
+        Ok(header)
+    }
+
     async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
         self.client
             .post(&format!("{}/tx", self.url))
@@ -277,31 +287,6 @@ impl UrlClient {
         Ok(result)
     }
 
-    async fn _script_list_unspent(
-        &self,
-        script: &Script,
-    ) -> Result<Vec<ELSListUnspentRes>, EsploraError> {
-        Ok(self
-            .client
-            .get(&format!(
-                "{}/scripthash/{}/utxo",
-                self.url,
-                Self::script_to_scripthash(script)
-            ))
-            .send()
-            .await?
-            .error_for_status()?
-            .json::<Vec<EsploraListUnspent>>()
-            .await?
-            .into_iter()
-            .map(|x| ELSListUnspentRes {
-                tx_hash: x.txid,
-                height: x.status.block_height.unwrap_or(0),
-                tx_pos: x.vout,
-            })
-            .collect())
-    }
-
     async fn _get_fee_estimates(&self) -> Result<HashMap<String, f64>, EsploraError> {
         Ok(self
             .client
@@ -321,32 +306,59 @@ impl ElectrumLikeSync for UrlClient {
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
         let future = async {
-            Ok(stream::iter(scripts)
-                .then(|script| self._script_get_history(&script))
-                .try_collect()
-                .await?)
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(scripts.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for script in chunk {
+                    futs.push(self._script_get_history(&script));
+                }
+                let partial_results: Vec<Vec<ELSGetHistoryRes>> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
         };
 
         await_or_block!(future)
     }
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
         let future = async {
-            Ok(stream::iter(scripts)
-                .then(|script| self._script_list_unspent(&script))
-                .try_collect()
-                .await?)
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(txids.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for txid in chunk {
+                    futs.push(self._get_tx_no_opt(&txid));
+                }
+                let partial_results: Vec<Transaction> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
         };
 
         await_or_block!(future)
     }
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error> {
-        Ok(await_or_block!(self._get_tx(txid))?
-            .ok_or_else(|| EsploraError::TransactionNotFound(*txid))?)
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        let future = async {
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(heights.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for height in chunk {
+                    futs.push(self._get_header(height));
+                }
+                let partial_results: Vec<BlockHeader> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
+        };
+
+        await_or_block!(future)
     }
 }
 
@@ -361,11 +373,26 @@ struct EsploraGetHistory {
     status: EsploraGetHistoryStatus,
 }
 
-#[derive(Deserialize)]
-struct EsploraListUnspent {
-    txid: Txid,
-    vout: usize,
-    status: EsploraGetHistoryStatus,
+/// Configuration for an [`EsploraBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct EsploraBlockchainConfig {
+    /// Base URL of the esplora service
+    ///
+    /// eg. `https://blockstream.info/api/`
+    pub base_url: String,
+    /// Number of parallel requests sent to the esplora service (default: 4)
+    pub concurrency: Option<u8>,
+}
+
+impl ConfigurableBlockchain for EsploraBlockchain {
+    type Config = EsploraBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(EsploraBlockchain::new(
+            config.base_url.as_str(),
+            config.concurrency,
+        ))
+    }
 }
 
 /// Errors that can happen during a sync with [`EsploraBlockchain`]
@@ -377,9 +404,15 @@ pub enum EsploraError {
     Parsing(std::num::ParseIntError),
     /// Invalid Bitcoin data returned
     BitcoinEncoding(bitcoin::consensus::encode::Error),
+    /// Invalid Hex data returned
+    Hex(bitcoin::hashes::hex::Error),
 
     /// Transaction not found
     TransactionNotFound(Txid),
+    /// Header height not found
+    HeaderHeightNotFound(u32),
+    /// Header hash not found
+    HeaderHashNotFound(BlockHash),
 }
 
 impl fmt::Display for EsploraError {
@@ -390,20 +423,7 @@ impl fmt::Display for EsploraError {
 
 impl std::error::Error for EsploraError {}
 
-impl From<reqwest::Error> for EsploraError {
-    fn from(other: reqwest::Error) -> Self {
-        EsploraError::Reqwest(other)
-    }
-}
-
-impl From<std::num::ParseIntError> for EsploraError {
-    fn from(other: std::num::ParseIntError) -> Self {
-        EsploraError::Parsing(other)
-    }
-}
-
-impl From<bitcoin::consensus::encode::Error> for EsploraError {
-    fn from(other: bitcoin::consensus::encode::Error) -> Self {
-        EsploraError::BitcoinEncoding(other)
-    }
-}
+impl_error!(reqwest::Error, Reqwest, EsploraError);
+impl_error!(std::num::ParseIntError, Parsing, EsploraError);
+impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
+impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);

src/blockchain/mod.rs

@@ -26,17 +26,8 @@
 //!
 //! This module provides the implementation of a few commonly-used backends like
 //! [Electrum](crate::blockchain::electrum), [Esplora](crate::blockchain::esplora) and
-//! [Compact Filters/Neutrino](crate::blockchain::compact_filters), along with two generalized
-//! traits [`Blockchain`] and [`OnlineBlockchain`] that can be implemented to build customized
-//! backends.
-//!
-//! Types that only implement the [`Blockchain`] trait can be used as backends for [`Wallet`](crate::wallet::Wallet)s, but any
-//! action that requires interacting with the blockchain won't be available ([`Wallet::sync`](crate::wallet::Wallet::sync) and
-//! [`Wallet::broadcast`](crate::wallet::Wallet::broadcast)). This allows the creation of physically air-gapped wallets, that have no
-//! ability to contact the outside world. An example of an offline-only client is [`OfflineBlockchain`].
-//!
-//! Types that also implement [`OnlineBlockchain`] will make the two aforementioned actions
-//! available.
+//! [Compact Filters/Neutrino](crate::blockchain::compact_filters), along with a generalized trait
+//! [`Blockchain`] that can be implemented to build customized backends.
 
 use std::collections::HashSet;
 use std::ops::Deref;
@@ -49,13 +40,21 @@ use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::FeeRate;
 
+#[cfg(any(feature = "electrum", feature = "esplora"))]
 pub(crate) mod utils;
 
+#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+pub mod any;
+#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+pub use any::{AnyBlockchain, AnyBlockchainConfig};
+
 #[cfg(feature = "electrum")]
 #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
 pub mod electrum;
 #[cfg(feature = "electrum")]
 pub use self::electrum::ElectrumBlockchain;
+#[cfg(feature = "electrum")]
+pub use self::electrum::ElectrumBlockchainConfig;
 
 #[cfg(feature = "esplora")]
 #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
@@ -69,7 +68,7 @@ pub mod compact_filters;
 #[cfg(feature = "compact_filters")]
 pub use self::compact_filters::CompactFiltersBlockchain;
 
-/// Capabilities that can be supported by an [`OnlineBlockchain`] backend
+/// Capabilities that can be supported by a [`Blockchain`] backend
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Capability {
     /// Can recover the full history of a wallet and not only the set of currently spendable UTXOs
@@ -80,56 +79,42 @@ pub enum Capability {
     AccurateFees,
 }
 
-/// Base trait for a blockchain backend
+/// Marker trait for a blockchain backend
 ///
-/// This trait is always required, even for "air-gapped" backends that don't actually make any
-/// external call. Clients that have the ability to make external calls must also implement `OnlineBlockchain`.
-pub trait Blockchain {
-    /// Return whether or not the client has the ability to fullfill requests
-    ///
-    /// This should always be `false` for offline-only types, and can be true for types that also
-    /// implement [`OnlineBlockchain`], if they have the ability to fullfill requests.
-    fn is_online(&self) -> bool;
+/// This is a marker trait for blockchain types. It is automatically implemented for types that
+/// implement [`Blockchain`], so as a user of the library you won't have to implement this
+/// manually.
+///
+/// Users of the library will probably never have to implement this trait manually, but they
+/// could still need to import it to define types and structs with generics;
+/// Implementing only the marker trait is pointless, since [`OfflineBlockchain`]
+/// already does that, and whenever [`Blockchain`] is implemented, the marker trait is also
+/// automatically implemented by the library.
+pub trait BlockchainMarker {}
 
-    /// Create a new instance of the client that is offline-only
-    ///
-    /// For types that also implement [`OnlineBlockchain`], this means creating an instance that
-    /// returns [`Error::OfflineClient`](crate::error::Error::OfflineClient) if any of the "online"
-    /// methods are called.
-    ///
-    /// This is generally implemented by wrapping the client in an [`Option`] that has [`Option::None`] value
-    /// when created with this method, and is [`Option::Some`] if properly instantiated.
-    fn offline() -> Self;
-}
+/// The [`BlockchainMarker`] marker trait is automatically implemented for [`Blockchain`] types
+impl<T: Blockchain> BlockchainMarker for T {}
 
-/// Type that only implements [`Blockchain`] and is always offline
+/// Type that only implements [`BlockchainMarker`] and is always "offline"
 pub struct OfflineBlockchain;
-impl Blockchain for OfflineBlockchain {
-    fn offline() -> Self {
-        OfflineBlockchain
-    }
+impl BlockchainMarker for OfflineBlockchain {}
 
-    fn is_online(&self) -> bool {
-        false
-    }
-}
-
-/// Trait that defines the actions that must be supported by an online [`Blockchain`]
+/// Trait that defines the actions that must be supported by a blockchain backend
 #[maybe_async]
-pub trait OnlineBlockchain: Blockchain {
+pub trait Blockchain: BlockchainMarker {
     /// Return the set of [`Capability`] supported by this backend
     fn get_capabilities(&self) -> HashSet<Capability>;
 
     /// Setup the backend and populate the internal database for the first time
     ///
-    /// This method is the equivalent of [`OnlineBlockchain::sync`], but it's guaranteed to only be
+    /// This method is the equivalent of [`Blockchain::sync`], but it's guaranteed to only be
     /// called once, at the first [`Wallet::sync`](crate::wallet::Wallet::sync).
     ///
     /// The rationale behind the distinction between `sync` and `setup` is that some custom backends
     /// might need to perform specific actions only the first time they are synced.
     ///
     /// For types that do not have that distinction, only this method can be implemented, since
-    /// [`OnlineBlockchain::sync`] defaults to calling this internally if not overridden.
+    /// [`Blockchain::sync`] defaults to calling this internally if not overridden.
     fn setup<D: BatchDatabase, P: 'static + Progress>(
         &self,
         stop_gap: Option<usize>,
@@ -138,7 +123,7 @@ pub trait OnlineBlockchain: Blockchain {
     ) -> Result<(), Error>;
     /// Populate the internal database with transactions and UTXOs
     ///
-    /// If not overridden, it defaults to calling [`OnlineBlockchain::setup`] internally.
+    /// If not overridden, it defaults to calling [`Blockchain::setup`] internally.
     ///
     /// This method should implement the logic required to iterate over the list of the wallet's
     /// script_pubkeys using [`Database::iter_script_pubkeys`] and look for relevant transactions
@@ -175,11 +160,20 @@ pub trait OnlineBlockchain: Blockchain {
     fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error>;
 }
 
+/// Trait for [`Blockchain`] types that can be created given a configuration
+pub trait ConfigurableBlockchain: Blockchain + Sized {
+    /// Type that contains the configuration
+    type Config: std::fmt::Debug;
+
+    /// Create a new instance given a configuration
+    fn from_config(config: &Self::Config) -> Result<Self, Error>;
+}
+
 /// Data sent with a progress update over a [`channel`]
 pub type ProgressData = (f32, Option<String>);
 
-/// Trait for types that can receive and process progress updates during [`OnlineBlockchain::sync`] and
-/// [`OnlineBlockchain::setup`]
+/// Trait for types that can receive and process progress updates during [`Blockchain::sync`] and
+/// [`Blockchain::setup`]
 pub trait Progress: Send {
     /// Send a new progress update
     ///
@@ -230,24 +224,18 @@ pub fn log_progress() -> LogProgress {
 
 impl Progress for LogProgress {
     fn update(&self, progress: f32, message: Option<String>) -> Result<(), Error> {
-        log::info!("Sync {:.3}%: `{}`", progress, message.unwrap_or("".into()));
+        log::info!(
+            "Sync {:.3}%: `{}`",
+            progress,
+            message.unwrap_or_else(|| "".into())
+        );
 
         Ok(())
     }
 }
 
-impl<T: Blockchain> Blockchain for Arc<T> {
-    fn is_online(&self) -> bool {
-        self.deref().is_online()
-    }
-
-    fn offline() -> Self {
-        Arc::new(T::offline())
-    }
-}
-
 #[maybe_async]
-impl<T: OnlineBlockchain> OnlineBlockchain for Arc<T> {
+impl<T: Blockchain> Blockchain for Arc<T> {
     fn get_capabilities(&self) -> HashSet<Capability> {
         maybe_await!(self.deref().get_capabilities())
     }

src/blockchain/utils.rs

@@ -22,19 +22,20 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-use std::cmp;
-use std::collections::{HashSet, VecDeque};
-use std::convert::TryFrom;
+use std::collections::{HashMap, HashSet};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
+use rand::seq::SliceRandom;
+use rand::thread_rng;
 
-use bitcoin::{Address, Network, OutPoint, Script, Transaction, Txid};
+use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 
 use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{ScriptType, TransactionDetails, UTXO};
+use crate::types::{KeychainKind, TransactionDetails, UTXO};
+use crate::wallet::time::Instant;
 use crate::wallet::utils::ChunksIterator;
 
 #[derive(Debug)]
@@ -43,316 +44,354 @@ pub struct ELSGetHistoryRes {
     pub tx_hash: Txid,
 }
 
-#[derive(Debug)]
-pub struct ELSListUnspentRes {
-    pub height: usize,
-    pub tx_hash: Txid,
-    pub tx_pos: usize,
-}
-
 /// Implements the synchronization logic for an Electrum-like client.
 #[maybe_async]
 pub trait ElectrumLikeSync {
-    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
         &self,
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error>;
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid> + Clone>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error>;
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error>;
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error>;
+    fn els_batch_block_header<I: IntoIterator<Item = u32> + Clone>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error>;
 
     // Provided methods down here...
 
     fn electrum_like_setup<D: BatchDatabase, P: Progress>(
         &self,
         stop_gap: Option<usize>,
-        database: &mut D,
+        db: &mut D,
         _progress_update: P,
     ) -> Result<(), Error> {
         // TODO: progress
+        let start = Instant::new();
+        debug!("start setup");
 
         let stop_gap = stop_gap.unwrap_or(20);
-        let batch_query_size = 20;
+        let chunk_size = stop_gap;
 
-        // check unconfirmed tx, delete so they are retrieved later
-        let mut del_batch = database.begin_batch();
-        for tx in database.iter_txs(false)? {
-            if tx.height.is_none() {
-                del_batch.del_tx(&tx.txid, false)?;
-            }
-        }
-        database.commit_batch(del_batch)?;
+        let mut history_txs_id = HashSet::new();
+        let mut txid_height = HashMap::new();
+        let mut max_indexes = HashMap::new();
 
-        // maximum derivation index for a change address that we've seen during sync
-        let mut change_max_deriv = None;
+        let mut wallet_chains = vec![KeychainKind::Internal, KeychainKind::External];
+        // shuffling improve privacy, the server doesn't know my first request is from my internal or external addresses
+        wallet_chains.shuffle(&mut thread_rng());
+        // download history of our internal and external script_pubkeys
+        for keychain in wallet_chains.iter() {
+            let script_iter = db.iter_script_pubkeys(Some(*keychain))?.into_iter();
 
-        let mut already_checked: HashSet<Script> = HashSet::new();
-        let mut to_check_later = VecDeque::with_capacity(batch_query_size);
-
-        // insert the first chunk
-        let mut iter_scriptpubkeys = database
-            .iter_script_pubkeys(Some(ScriptType::External))?
-            .into_iter();
-        let chunk: Vec<Script> = iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-        for item in chunk.into_iter().rev() {
-            to_check_later.push_front(item);
-        }
-
-        let mut iterating_external = true;
-        let mut index = 0;
-        let mut last_found = None;
-        while !to_check_later.is_empty() {
-            trace!("to_check_later size {}", to_check_later.len());
-
-            let until = cmp::min(to_check_later.len(), batch_query_size);
-            let chunk: Vec<Script> = to_check_later.drain(..until).collect();
-            let call_result = maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
-
-            for (script, history) in chunk.into_iter().zip(call_result.into_iter()) {
-                trace!("received history for {:?}, size {}", script, history.len());
-
-                if !history.is_empty() {
-                    last_found = Some(index);
-
-                    let mut check_later_scripts = maybe_await!(self.check_history(
-                        database,
-                        script,
-                        history,
-                        &mut change_max_deriv
-                    ))?
-                    .into_iter()
-                    .filter(|x| already_checked.insert(x.clone()))
-                    .collect();
-                    to_check_later.append(&mut check_later_scripts);
+            for (i, chunk) in ChunksIterator::new(script_iter, stop_gap).enumerate() {
+                // TODO if i == last, should create another chunk of addresses in db
+                let call_result: Vec<Vec<ELSGetHistoryRes>> =
+                    maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
+                let max_index = call_result
+                    .iter()
+                    .enumerate()
+                    .filter_map(|(i, v)| v.first().map(|_| i as u32))
+                    .max();
+                if let Some(max) = max_index {
+                    max_indexes.insert(keychain, max + (i * chunk_size) as u32);
+                }
+                let flattened: Vec<ELSGetHistoryRes> = call_result.into_iter().flatten().collect();
+                debug!("#{} of {:?} results:{}", i, keychain, flattened.len());
+                if flattened.is_empty() {
+                    // Didn't find anything in the last `stop_gap` script_pubkeys, breaking
+                    break;
                 }
 
-                index += 1;
-            }
-
-            match iterating_external {
-                true if index - last_found.unwrap_or(0) >= stop_gap => iterating_external = false,
-                true => {
-                    trace!("pushing one more batch from `iter_scriptpubkeys`. index = {}, last_found = {:?}, stop_gap = {}", index, last_found, stop_gap);
-
-                    let chunk: Vec<Script> =
-                        iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-                    for item in chunk.into_iter().rev() {
-                        to_check_later.push_front(item);
+                for el in flattened {
+                    // el.height = -1 means unconfirmed with unconfirmed parents
+                    // el.height =  0 means unconfirmed with confirmed parents
+                    // but we treat those tx the same
+                    if el.height <= 0 {
+                        txid_height.insert(el.tx_hash, None);
+                    } else {
+                        txid_height.insert(el.tx_hash, Some(el.height as u32));
                     }
-                }
-                _ => {}
-            }
-        }
-
-        // check utxo
-        // TODO: try to minimize network requests and re-use scripts if possible
-        let mut batch = database.begin_batch();
-        for chunk in ChunksIterator::new(database.iter_utxos()?.into_iter(), batch_query_size) {
-            let scripts: Vec<_> = chunk.iter().map(|u| &u.txout.script_pubkey).collect();
-            let call_result = maybe_await!(self.els_batch_script_list_unspent(scripts))?;
-
-            // check which utxos are actually still unspent
-            for (utxo, list_unspent) in chunk.into_iter().zip(call_result.iter()) {
-                debug!(
-                    "outpoint {:?} is unspent for me, list unspent is {:?}",
-                    utxo.outpoint, list_unspent
-                );
-
-                let mut spent = true;
-                for unspent in list_unspent {
-                    let res_outpoint = OutPoint::new(unspent.tx_hash, unspent.tx_pos as u32);
-                    if utxo.outpoint == res_outpoint {
-                        spent = false;
-                        break;
-                    }
-                }
-                if spent {
-                    info!("{} not anymore unspent, removing", utxo.outpoint);
-                    batch.del_utxo(&utxo.outpoint)?;
+                    history_txs_id.insert(el.tx_hash);
                 }
             }
         }
 
-        let current_ext = database.get_last_index(ScriptType::External)?.unwrap_or(0);
-        let first_ext_new = last_found.map(|x| x + 1).unwrap_or(0) as u32;
-        if first_ext_new > current_ext {
-            info!("Setting external index to {}", first_ext_new);
-            database.set_last_index(ScriptType::External, first_ext_new)?;
+        // saving max indexes
+        info!("max indexes are: {:?}", max_indexes);
+        for keychain in wallet_chains.iter() {
+            if let Some(index) = max_indexes.get(keychain) {
+                db.set_last_index(*keychain, *index)?;
+            }
         }
 
-        let current_int = database.get_last_index(ScriptType::Internal)?.unwrap_or(0);
-        let first_int_new = change_max_deriv.map(|x| x + 1).unwrap_or(0);
-        if first_int_new > current_int {
-            info!("Setting internal index to {}", first_int_new);
-            database.set_last_index(ScriptType::Internal, first_int_new)?;
+        // get db status
+        let txs_details_in_db: HashMap<Txid, TransactionDetails> = db
+            .iter_txs(false)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let txs_raw_in_db: HashMap<Txid, Transaction> = db
+            .iter_raw_txs()?
+            .into_iter()
+            .map(|tx| (tx.txid(), tx))
+            .collect();
+        let utxos_deps = utxos_deps(db, &txs_raw_in_db)?;
+
+        // download new txs and headers
+        let new_txs = maybe_await!(self.download_and_save_needed_raw_txs(
+            &history_txs_id,
+            &txs_raw_in_db,
+            chunk_size,
+            db
+        ))?;
+        let new_timestamps = maybe_await!(self.download_needed_headers(
+            &txid_height,
+            &txs_details_in_db,
+            chunk_size
+        ))?;
+
+        let mut batch = db.begin_batch();
+
+        // save any tx details not in db but in history_txs_id or with different height/timestamp
+        for txid in history_txs_id.iter() {
+            let height = txid_height.get(txid).cloned().flatten();
+            let timestamp = *new_timestamps.get(txid).unwrap_or(&0u64);
+            if let Some(tx_details) = txs_details_in_db.get(txid) {
+                // check if height matches, otherwise updates it
+                if tx_details.height != height {
+                    let mut new_tx_details = tx_details.clone();
+                    new_tx_details.height = height;
+                    new_tx_details.timestamp = timestamp;
+                    batch.set_tx(&new_tx_details)?;
+                }
+            } else {
+                save_transaction_details_and_utxos(
+                    &txid,
+                    db,
+                    timestamp,
+                    height,
+                    &mut batch,
+                    &utxos_deps,
+                )?;
+            }
         }
 
-        database.commit_batch(batch)?;
+        // remove any tx details in db but not in history_txs_id
+        for txid in txs_details_in_db.keys() {
+            if !history_txs_id.contains(txid) {
+                batch.del_tx(&txid, false)?;
+            }
+        }
+
+        // remove any spent utxo
+        for new_tx in new_txs.iter() {
+            for input in new_tx.input.iter() {
+                batch.del_utxo(&input.previous_output)?;
+            }
+        }
+
+        db.commit_batch(batch)?;
+        info!("finish setup, elapsed {:?}ms", start.elapsed().as_millis());
 
         Ok(())
     }
 
-    fn check_tx_and_descendant<D: BatchDatabase>(
+    /// download txs identified by `history_txs_id` and theirs previous outputs if not already present in db
+    fn download_and_save_needed_raw_txs<D: BatchDatabase>(
         &self,
-        database: &mut D,
-        txid: &Txid,
-        height: Option<u32>,
-        cur_script: &Script,
-        change_max_deriv: &mut Option<u32>,
-    ) -> Result<Vec<Script>, Error> {
-        debug!(
-            "check_tx_and_descendant of {}, height: {:?}, script: {}",
-            txid, height, cur_script
-        );
-        let mut updates = database.begin_batch();
-        let tx = match database.get_tx(&txid, true)? {
-            Some(mut saved_tx) => {
-                // update the height if it's different (in case of reorg)
-                if saved_tx.height != height {
-                    info!(
-                        "updating height from {:?} to {:?} for tx {}",
-                        saved_tx.height, height, txid
-                    );
-                    saved_tx.height = height;
-                    updates.set_tx(&saved_tx)?;
-                }
-
-                debug!("already have {} in db, returning the cached version", txid);
-
-                // unwrap since we explicitly ask for the raw_tx, if it's not present something
-                // went wrong
-                saved_tx.transaction.unwrap()
-            }
-            None => {
-                let fetched_tx = maybe_await!(self.els_transaction_get(&txid))?;
-                database.set_raw_tx(&fetched_tx)?;
-
-                fetched_tx
-            }
-        };
-
-        let mut incoming: u64 = 0;
-        let mut outgoing: u64 = 0;
-
-        let mut inputs_sum: u64 = 0;
-        let mut outputs_sum: u64 = 0;
-
-        // look for our own inputs
-        for (i, input) in tx.input.iter().enumerate() {
-            // the fact that we visit addresses in a BFS fashion starting from the external addresses
-            // should ensure that this query is always consistent (i.e. when we get to call this all
-            // the transactions at a lower depth have already been indexed, so if an outpoint is ours
-            // we are guaranteed to have it in the db).
-            if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
-                inputs_sum += previous_output.value;
-
-                if database.is_mine(&previous_output.script_pubkey)? {
-                    outgoing += previous_output.value;
-
-                    debug!("{} input #{} is mine, removing from utxo", txid, i);
-                    updates.del_utxo(&input.previous_output)?;
-                }
-            } else {
-                // The input is not ours, but we still need to count it for the fees. so fetch the
-                // tx (from the database or from network) and check it
-                let tx = match database.get_tx(&input.previous_output.txid, true)? {
-                    Some(saved_tx) => saved_tx.transaction.unwrap(),
-                    None => {
-                        let fetched_tx =
-                            maybe_await!(self.els_transaction_get(&input.previous_output.txid))?;
-                        database.set_raw_tx(&fetched_tx)?;
-
-                        fetched_tx
-                    }
-                };
-
-                inputs_sum += tx.output[input.previous_output.vout as usize].value;
-            }
-        }
-
-        let mut to_check_later = vec![];
-        for (i, output) in tx.output.iter().enumerate() {
-            // to compute the fees later
-            outputs_sum += output.value;
-
-            // this output is ours, we have a path to derive it
-            if let Some((script_type, child)) =
-                database.get_path_from_script_pubkey(&output.script_pubkey)?
-            {
-                debug!("{} output #{} is mine, adding utxo", txid, i);
-                updates.set_utxo(&UTXO {
-                    outpoint: OutPoint::new(tx.txid(), i as u32),
-                    txout: output.clone(),
-                    is_internal: script_type.is_internal(),
-                })?;
-                incoming += output.value;
-
-                if output.script_pubkey != *cur_script {
-                    debug!("{} output #{} script {} was not current script, adding script to be checked later", txid, i, output.script_pubkey);
-                    to_check_later.push(output.script_pubkey.clone())
-                }
-
-                // derive as many change addrs as external addresses that we've seen
-                if script_type == ScriptType::Internal
-                    && (change_max_deriv.is_none() || child > change_max_deriv.unwrap_or(0))
-                {
-                    *change_max_deriv = Some(child);
+        history_txs_id: &HashSet<Txid>,
+        txs_raw_in_db: &HashMap<Txid, Transaction>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        let txids_raw_in_db: HashSet<Txid> = txs_raw_in_db.keys().cloned().collect();
+        let txids_to_download: Vec<&Txid> = history_txs_id.difference(&txids_raw_in_db).collect();
+        if !txids_to_download.is_empty() {
+            info!("got {} txs to download", txids_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                txids_to_download,
+                chunk_size,
+                db,
+            ))?);
+            let mut prev_txids = HashSet::new();
+            let mut txids_downloaded = HashSet::new();
+            for tx in txs_downloaded.iter() {
+                txids_downloaded.insert(tx.txid());
+                // add every previous input tx, but skip coinbase
+                for input in tx.input.iter().filter(|i| !i.previous_output.is_null()) {
+                    prev_txids.insert(input.previous_output.txid);
                 }
             }
-        }
-
-        let tx = TransactionDetails {
-            txid: tx.txid(),
-            transaction: Some(tx),
-            received: incoming,
-            sent: outgoing,
-            height,
-            timestamp: 0,
-            fees: inputs_sum - outputs_sum,
-        };
-        info!("Saving tx {}", txid);
-        updates.set_tx(&tx)?;
-
-        database.commit_batch(updates)?;
-
-        Ok(to_check_later)
-    }
-
-    fn check_history<D: BatchDatabase>(
-        &self,
-        database: &mut D,
-        script_pubkey: Script,
-        txs: Vec<ELSGetHistoryRes>,
-        change_max_deriv: &mut Option<u32>,
-    ) -> Result<Vec<Script>, Error> {
-        let mut to_check_later = Vec::new();
-
-        debug!(
-            "history of {} script {} has {} tx",
-            Address::from_script(&script_pubkey, Network::Testnet).unwrap(),
-            script_pubkey,
-            txs.len()
-        );
-
-        for tx in txs {
-            let height: Option<u32> = match tx.height {
-                0 | -1 => None,
-                x => u32::try_from(x).ok(),
-            };
-
-            to_check_later.extend_from_slice(&maybe_await!(self.check_tx_and_descendant(
-                database,
-                &tx.tx_hash,
-                height,
-                &script_pubkey,
-                change_max_deriv,
+            let already_present: HashSet<Txid> =
+                txids_downloaded.union(&txids_raw_in_db).cloned().collect();
+            let prev_txs_to_download: Vec<&Txid> =
+                prev_txids.difference(&already_present).collect();
+            info!("{} previous txs to download", prev_txs_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                prev_txs_to_download,
+                chunk_size,
+                db,
             ))?);
         }
 
-        Ok(to_check_later)
+        Ok(txs_downloaded)
+    }
+
+    /// download headers at heights in `txid_height` if tx details not already present, returns a map Txid -> timestamp
+    fn download_needed_headers(
+        &self,
+        txid_height: &HashMap<Txid, Option<u32>>,
+        txs_details_in_db: &HashMap<Txid, TransactionDetails>,
+        chunk_size: usize,
+    ) -> Result<HashMap<Txid, u64>, Error> {
+        let mut txid_timestamp = HashMap::new();
+        let needed_txid_height: HashMap<&Txid, u32> = txid_height
+            .iter()
+            .filter(|(t, _)| txs_details_in_db.get(*t).is_none())
+            .filter_map(|(t, o)| o.map(|h| (t, h)))
+            .collect();
+        let needed_heights: HashSet<u32> = needed_txid_height.values().cloned().collect();
+        if !needed_heights.is_empty() {
+            info!("{} headers to download for timestamp", needed_heights.len());
+            let mut height_timestamp: HashMap<u32, u64> = HashMap::new();
+            for chunk in ChunksIterator::new(needed_heights.into_iter(), chunk_size) {
+                let call_result: Vec<BlockHeader> =
+                    maybe_await!(self.els_batch_block_header(chunk.clone()))?;
+                height_timestamp.extend(
+                    chunk
+                        .into_iter()
+                        .zip(call_result.iter().map(|h| h.time as u64)),
+                );
+            }
+            for (txid, height) in needed_txid_height {
+                let timestamp = height_timestamp
+                    .get(&height)
+                    .ok_or_else(|| Error::Generic("timestamp missing".to_string()))?;
+                txid_timestamp.insert(*txid, *timestamp);
+            }
+        }
+
+        Ok(txid_timestamp)
+    }
+
+    fn download_and_save_in_chunks<D: BatchDatabase>(
+        &self,
+        to_download: Vec<&Txid>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        for chunk in ChunksIterator::new(to_download.into_iter(), chunk_size) {
+            let call_result: Vec<Transaction> =
+                maybe_await!(self.els_batch_transaction_get(chunk))?;
+            let mut batch = db.begin_batch();
+            for new_tx in call_result.iter() {
+                batch.set_raw_tx(new_tx)?;
+            }
+            db.commit_batch(batch)?;
+            txs_downloaded.extend(call_result);
+        }
+
+        Ok(txs_downloaded)
     }
 }
+
+fn save_transaction_details_and_utxos<D: BatchDatabase>(
+    txid: &Txid,
+    db: &mut D,
+    timestamp: u64,
+    height: Option<u32>,
+    updates: &mut dyn BatchOperations,
+    utxo_deps: &HashMap<OutPoint, OutPoint>,
+) -> Result<(), Error> {
+    let tx = db.get_raw_tx(txid)?.ok_or(Error::TransactionNotFound)?;
+
+    let mut incoming: u64 = 0;
+    let mut outgoing: u64 = 0;
+
+    let mut inputs_sum: u64 = 0;
+    let mut outputs_sum: u64 = 0;
+
+    // look for our own inputs
+    for input in tx.input.iter() {
+        // skip coinbase inputs
+        if input.previous_output.is_null() {
+            continue;
+        }
+
+        // We already downloaded all previous output txs in the previous step
+        if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+            inputs_sum += previous_output.value;
+
+            if db.is_mine(&previous_output.script_pubkey)? {
+                outgoing += previous_output.value;
+            }
+        } else {
+            // The input is not ours, but we still need to count it for the fees
+            let tx = db
+                .get_raw_tx(&input.previous_output.txid)?
+                .ok_or(Error::TransactionNotFound)?;
+            inputs_sum += tx.output[input.previous_output.vout as usize].value;
+        }
+
+        // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
+        if let Some(outpoint) = utxo_deps.get(&input.previous_output) {
+            updates.del_utxo(&outpoint)?;
+        }
+    }
+
+    for (i, output) in tx.output.iter().enumerate() {
+        // to compute the fees later
+        outputs_sum += output.value;
+
+        // this output is ours, we have a path to derive it
+        if let Some((keychain, _child)) = db.get_path_from_script_pubkey(&output.script_pubkey)? {
+            debug!("{} output #{} is mine, adding utxo", txid, i);
+            updates.set_utxo(&UTXO {
+                outpoint: OutPoint::new(tx.txid(), i as u32),
+                txout: output.clone(),
+                keychain,
+            })?;
+
+            incoming += output.value;
+        }
+    }
+
+    let tx_details = TransactionDetails {
+        txid: tx.txid(),
+        transaction: Some(tx),
+        received: incoming,
+        sent: outgoing,
+        height,
+        timestamp,
+        fees: inputs_sum.saturating_sub(outputs_sum), /* if the tx is a coinbase, fees would be negative */
+    };
+    updates.set_tx(&tx_details)?;
+
+    Ok(())
+}
+
+/// returns utxo dependency as the inputs needed for the utxo to exist
+/// `tx_raw_in_db` must contains utxo's generating txs or errors witt [crate::Error::TransactionNotFound]
+fn utxos_deps<D: BatchDatabase>(
+    db: &mut D,
+    tx_raw_in_db: &HashMap<Txid, Transaction>,
+) -> Result<HashMap<OutPoint, OutPoint>, Error> {
+    let utxos = db.iter_utxos()?;
+    let mut utxos_deps = HashMap::new();
+    for utxo in utxos {
+        let from_tx = tx_raw_in_db
+            .get(&utxo.outpoint.txid)
+            .ok_or(Error::TransactionNotFound)?;
+        for input in from_tx.input.iter() {
+            utxos_deps.insert(input.previous_output, utxo.outpoint);
+        }
+    }
+    Ok(utxos_deps)
+}

src/database/any.rs

@@ -0,0 +1,388 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Runtime-checked database types
+//!
+//! This module provides the implementation of [`AnyDatabase`] which allows switching the
+//! inner [`Database`] type at runtime.
+//!
+//! ## Example
+//!
+//! In this example, `wallet_memory` and `wallet_sled` have the same type of `Wallet<OfflineBlockchain, AnyDatabase>`.
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::database::{AnyDatabase, MemoryDatabase};
+//! # use bdk::{Wallet, OfflineWallet};
+//! let memory = MemoryDatabase::default().into();
+//! let wallet_memory: OfflineWallet<AnyDatabase> =
+//!     Wallet::new_offline("...", None, Network::Testnet, memory)?;
+//!
+//! # #[cfg(feature = "key-value-db")]
+//! # {
+//! let sled = sled::open("my-database")?.open_tree("default_tree")?.into();
+//! let wallet_sled: OfflineWallet<AnyDatabase> =
+//!     Wallet::new_offline("...", None, Network::Testnet, sled)?;
+//! # }
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! When paired with the use of [`ConfigurableDatabase`], it allows creating wallets with any
+//! database supported using a single line of code:
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::database::*;
+//! # use bdk::{Wallet, OfflineWallet};
+//! let config = serde_json::from_str("...")?;
+//! let database = AnyDatabase::from_config(&config)?;
+//! let wallet: OfflineWallet<_> = Wallet::new_offline("...", None, Network::Testnet, database)?;
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use super::*;
+
+macro_rules! impl_from {
+    ( $from:ty, $to:ty, $variant:ident, $( $cfg:tt )* ) => {
+        $( $cfg )*
+        impl From<$from> for $to {
+            fn from(inner: $from) -> Self {
+                <$to>::$variant(inner)
+            }
+        }
+    };
+}
+
+macro_rules! impl_inner_method {
+    ( $enum_name:ident, $self:expr, $name:ident $(, $args:expr)* ) => {
+        match $self {
+            $enum_name::Memory(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "key-value-db")]
+            $enum_name::Sled(inner) => inner.$name( $($args, )* ),
+        }
+    }
+}
+
+/// Type that can contain any of the [`Database`] types defined by the library
+///
+/// It allows switching database type at runtime.
+///
+/// See [this module](crate::database::any)'s documentation for a usage example.
+#[derive(Debug)]
+pub enum AnyDatabase {
+    /// In-memory ephemeral database
+    Memory(memory::MemoryDatabase),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(sled::Tree),
+}
+
+impl_from!(memory::MemoryDatabase, AnyDatabase, Memory,);
+impl_from!(sled::Tree, AnyDatabase, Sled, #[cfg(feature = "key-value-db")]);
+
+/// Type that contains any of the [`BatchDatabase::Batch`] types defined by the library
+pub enum AnyBatch {
+    /// In-memory ephemeral database
+    Memory(<memory::MemoryDatabase as BatchDatabase>::Batch),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(<sled::Tree as BatchDatabase>::Batch),
+}
+
+impl_from!(
+    <memory::MemoryDatabase as BatchDatabase>::Batch,
+    AnyBatch,
+    Memory,
+);
+impl_from!(<sled::Tree as BatchDatabase>::Batch, AnyBatch, Sled, #[cfg(feature = "key-value-db")]);
+
+impl BatchOperations for AnyDatabase {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            set_script_pubkey,
+            script,
+            keychain,
+            child
+        )
+    }
+    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_utxo, utxo)
+    }
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_raw_tx, transaction)
+    }
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_tx, transaction)
+    }
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_last_index, keychain, value)
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            del_script_pubkey_from_path,
+            keychain,
+            child
+        )
+    }
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_path_from_script_pubkey, script)
+    }
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_utxo, outpoint)
+    }
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_raw_tx, txid)
+    }
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_tx, txid, include_raw)
+    }
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_last_index, keychain)
+    }
+}
+
+impl Database for AnyDatabase {
+    fn check_descriptor_checksum<B: AsRef<[u8]>>(
+        &mut self,
+        keychain: KeychainKind,
+        bytes: B,
+    ) -> Result<(), Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            check_descriptor_checksum,
+            keychain,
+            bytes
+        )
+    }
+
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_script_pubkeys, keychain)
+    }
+    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_utxos)
+    }
+    fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_raw_txs)
+    }
+    fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_txs, include_raw)
+    }
+
+    fn get_script_pubkey_from_path(
+        &self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            get_script_pubkey_from_path,
+            keychain,
+            child
+        )
+    }
+    fn get_path_from_script_pubkey(
+        &self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_path_from_script_pubkey, script)
+    }
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_utxo, outpoint)
+    }
+    fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_raw_tx, txid)
+    }
+    fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_tx, txid, include_raw)
+    }
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_last_index, keychain)
+    }
+
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        impl_inner_method!(AnyDatabase, self, increment_last_index, keychain)
+    }
+}
+
+impl BatchOperations for AnyBatch {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_script_pubkey, script, keychain, child)
+    }
+    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_utxo, utxo)
+    }
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_raw_tx, transaction)
+    }
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_tx, transaction)
+    }
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_last_index, keychain, value)
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(AnyBatch, self, del_script_pubkey_from_path, keychain, child)
+    }
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyBatch, self, del_path_from_script_pubkey, script)
+    }
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+        impl_inner_method!(AnyBatch, self, del_utxo, outpoint)
+    }
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyBatch, self, del_raw_tx, txid)
+    }
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyBatch, self, del_tx, txid, include_raw)
+    }
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyBatch, self, del_last_index, keychain)
+    }
+}
+
+impl BatchDatabase for AnyDatabase {
+    type Batch = AnyBatch;
+
+    fn begin_batch(&self) -> Self::Batch {
+        match self {
+            AnyDatabase::Memory(inner) => inner.begin_batch().into(),
+            #[cfg(feature = "key-value-db")]
+            AnyDatabase::Sled(inner) => inner.begin_batch().into(),
+        }
+    }
+    fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> {
+        // TODO: refactor once `move_ref_pattern` is stable
+        #[allow(irrefutable_let_patterns)]
+        match self {
+            AnyDatabase::Memory(db) => {
+                if let AnyBatch::Memory(batch) = batch {
+                    db.commit_batch(batch)
+                } else {
+                    unimplemented!()
+                }
+            }
+            #[cfg(feature = "key-value-db")]
+            AnyDatabase::Sled(db) => {
+                if let AnyBatch::Sled(batch) = batch {
+                    db.commit_batch(batch)
+                } else {
+                    unimplemented!()
+                }
+            }
+        }
+    }
+}
+
+/// Configuration type for a [`sled::Tree`] database
+#[cfg(feature = "key-value-db")]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct SledDbConfiguration {
+    /// Main directory of the db
+    pub path: String,
+    /// Name of the database tree, a separated namespace for the data
+    pub tree_name: String,
+}
+
+#[cfg(feature = "key-value-db")]
+impl ConfigurableDatabase for sled::Tree {
+    type Config = SledDbConfiguration;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(sled::open(&config.path)?.open_tree(&config.tree_name)?)
+    }
+}
+
+/// Type that can contain any of the database configurations defined by the library
+///
+/// This allows storing a single configuration that can be loaded into an [`AnyDatabase`]
+/// instance. Wallets that plan to offer users the ability to switch blockchain backend at runtime
+/// will find this particularly useful.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub enum AnyDatabaseConfig {
+    /// Memory database has no config
+    Memory(()),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(SledDbConfiguration),
+}
+
+impl ConfigurableDatabase for AnyDatabase {
+    type Config = AnyDatabaseConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(match config {
+            AnyDatabaseConfig::Memory(inner) => {
+                AnyDatabase::Memory(memory::MemoryDatabase::from_config(inner)?)
+            }
+            #[cfg(feature = "key-value-db")]
+            AnyDatabaseConfig::Sled(inner) => AnyDatabase::Sled(sled::Tree::from_config(inner)?),
+        })
+    }
+}
+
+impl_from!((), AnyDatabaseConfig, Memory,);
+impl_from!(SledDbConfiguration, AnyDatabaseConfig, Sled, #[cfg(feature = "key-value-db")]);

src/database/keyvalue.rs

@@ -37,13 +37,13 @@ use crate::types::*;
 
 macro_rules! impl_batch_operations {
     ( { $($after_insert:tt)* }, $process_delete:ident ) => {
-        fn set_script_pubkey(&mut self, script: &Script, script_type: ScriptType, path: u32) -> Result<(), Error> {
-            let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        fn set_script_pubkey(&mut self, script: &Script, keychain: KeychainKind, path: u32) -> Result<(), Error> {
+            let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
             self.insert(key, serialize(script))$($after_insert)*;
 
             let key = MapKey::Script(Some(script)).as_map_key();
             let value = json!({
-                "t": script_type,
+                "t": keychain,
                 "p": path,
             });
             self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
@@ -55,7 +55,7 @@ macro_rules! impl_batch_operations {
             let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
             let value = json!({
                 "t": utxo.txout,
-                "i": utxo.is_internal,
+                "i": utxo.keychain,
             });
             self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
 
@@ -88,22 +88,22 @@ macro_rules! impl_batch_operations {
             Ok(())
         }
 
-        fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error> {
-            let key = MapKey::LastIndex(script_type).as_map_key();
+        fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+            let key = MapKey::LastIndex(keychain).as_map_key();
             self.insert(key, &value.to_be_bytes())$($after_insert)*;
 
             Ok(())
         }
 
-        fn del_script_pubkey_from_path(&mut self, script_type: ScriptType, path: u32) -> Result<Option<Script>, Error> {
-            let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        fn del_script_pubkey_from_path(&mut self, keychain: KeychainKind, path: u32) -> Result<Option<Script>, Error> {
+            let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
             Ok(res.map_or(Ok(None), |x| Some(deserialize(&x)).transpose())?)
         }
 
-        fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result<Option<(ScriptType, u32)>, Error> {
+        fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result<Option<(KeychainKind, u32)>, Error> {
             let key = MapKey::Script(Some(script)).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
@@ -130,9 +130,9 @@ macro_rules! impl_batch_operations {
                 Some(b) => {
                     let mut val: serde_json::Value = serde_json::from_slice(&b)?;
                     let txout = serde_json::from_value(val["t"].take())?;
-                    let is_internal = serde_json::from_value(val["i"].take())?;
+                    let keychain = serde_json::from_value(val["i"].take())?;
 
-                    Ok(Some(UTXO { outpoint: outpoint.clone(), txout, is_internal }))
+                    Ok(Some(UTXO { outpoint: outpoint.clone(), txout, keychain }))
                 }
             }
         }
@@ -167,8 +167,8 @@ macro_rules! impl_batch_operations {
             }
         }
 
-        fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-            let key = MapKey::LastIndex(script_type).as_map_key();
+        fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+            let key = MapKey::LastIndex(keychain).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
@@ -206,10 +206,10 @@ impl BatchOperations for Batch {
 impl Database for Tree {
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error> {
-        let key = MapKey::DescriptorChecksum(script_type).as_map_key();
+        let key = MapKey::DescriptorChecksum(keychain).as_map_key();
 
         let prev = self.get(&key)?.map(|x| x.to_vec());
         if let Some(val) = prev {
@@ -224,8 +224,8 @@ impl Database for Tree {
         }
     }
 
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error> {
-        let key = MapKey::Path((script_type, None)).as_map_key();
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        let key = MapKey::Path((keychain, None)).as_map_key();
         self.scan_prefix(key)
             .map(|x| -> Result<_, Error> {
                 let (_, v) = x?;
@@ -243,12 +243,12 @@ impl Database for Tree {
 
                 let mut val: serde_json::Value = serde_json::from_slice(&v)?;
                 let txout = serde_json::from_value(val["t"].take())?;
-                let is_internal = serde_json::from_value(val["i"].take())?;
+                let keychain = serde_json::from_value(val["i"].take())?;
 
                 Ok(UTXO {
                     outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 })
             })
             .collect()
@@ -282,17 +282,17 @@ impl Database for Tree {
 
     fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<Option<Script>, Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         Ok(self.get(key)?.map(|b| deserialize(&b)).transpose()?)
     }
 
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
@@ -311,12 +311,12 @@ impl Database for Tree {
             .map(|b| -> Result<_, Error> {
                 let mut val: serde_json::Value = serde_json::from_slice(&b)?;
                 let txout = serde_json::from_value(val["t"].take())?;
-                let is_internal = serde_json::from_value(val["i"].take())?;
+                let keychain = serde_json::from_value(val["i"].take())?;
 
                 Ok(UTXO {
-                    outpoint: outpoint.clone(),
+                    outpoint: *outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 })
             })
             .transpose()
@@ -341,8 +341,8 @@ impl Database for Tree {
             .transpose()
     }
 
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
                 let array: [u8; 4] = b
@@ -356,8 +356,8 @@ impl Database for Tree {
     }
 
     // inserts 0 if not present
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.update_and_fetch(key, |prev| {
             let new = match prev {
                 Some(b) => {

src/database/memory.rs

@@ -34,7 +34,7 @@ use bitcoin::consensus::encode::{deserialize, serialize};
 use bitcoin::hash_types::Txid;
 use bitcoin::{OutPoint, Script, Transaction};
 
-use crate::database::{BatchDatabase, BatchOperations, Database};
+use crate::database::{BatchDatabase, BatchOperations, ConfigurableDatabase, Database};
 use crate::error::Error;
 use crate::types::*;
 
@@ -47,17 +47,17 @@ use crate::types::*;
 // descriptor checksum  d{i,e} -> vec<u8>
 
 pub(crate) enum MapKey<'a> {
-    Path((Option<ScriptType>, Option<u32>)),
+    Path((Option<KeychainKind>, Option<u32>)),
     Script(Option<&'a Script>),
     UTXO(Option<&'a OutPoint>),
     RawTx(Option<&'a Txid>),
     Transaction(Option<&'a Txid>),
-    LastIndex(ScriptType),
-    DescriptorChecksum(ScriptType),
+    LastIndex(KeychainKind),
+    DescriptorChecksum(KeychainKind),
 }
 
 impl MapKey<'_> {
-    pub fn as_prefix(&self) -> Vec<u8> {
+    fn as_prefix(&self) -> Vec<u8> {
         match self {
             MapKey::Path((st, _)) => {
                 let mut v = b"p".to_vec();
@@ -77,7 +77,7 @@ impl MapKey<'_> {
 
     fn serialize_content(&self) -> Vec<u8> {
         match self {
-            MapKey::Path((_, Some(child))) => u32::from(*child).to_be_bytes().to_vec(),
+            MapKey::Path((_, Some(child))) => child.to_be_bytes().to_vec(),
             MapKey::Script(Some(s)) => serialize(*s),
             MapKey::UTXO(Some(s)) => serialize(*s),
             MapKey::RawTx(Some(s)) => serialize(*s),
@@ -94,8 +94,8 @@ impl MapKey<'_> {
     }
 }
 
-fn after(key: &Vec<u8>) -> Vec<u8> {
-    let mut key = key.clone();
+fn after(key: &[u8]) -> Vec<u8> {
+    let mut key = key.to_owned();
     let mut idx = key.len();
     while idx > 0 {
         if key[idx - 1] == 0xFF {
@@ -141,15 +141,15 @@ impl BatchOperations for MemoryDatabase {
     fn set_script_pubkey(
         &mut self,
         script: &Script,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<(), Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         self.map.insert(key, Box::new(script.clone()));
 
         let key = MapKey::Script(Some(script)).as_map_key();
         let value = json!({
-            "t": script_type,
+            "t": keychain,
             "p": path,
         });
         self.map.insert(key, Box::new(value));
@@ -160,7 +160,7 @@ impl BatchOperations for MemoryDatabase {
     fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
         let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
         self.map
-            .insert(key, Box::new((utxo.txout.clone(), utxo.is_internal)));
+            .insert(key, Box::new((utxo.txout.clone(), utxo.keychain)));
 
         Ok(())
     }
@@ -186,8 +186,8 @@ impl BatchOperations for MemoryDatabase {
 
         Ok(())
     }
-    fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.map.insert(key, Box::new(value));
 
         Ok(())
@@ -195,10 +195,10 @@ impl BatchOperations for MemoryDatabase {
 
     fn del_script_pubkey_from_path(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<Option<Script>, Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
@@ -207,7 +207,7 @@ impl BatchOperations for MemoryDatabase {
     fn del_path_from_script_pubkey(
         &mut self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
@@ -231,11 +231,11 @@ impl BatchOperations for MemoryDatabase {
         match res {
             None => Ok(None),
             Some(b) => {
-                let (txout, is_internal) = b.downcast_ref().cloned().unwrap();
+                let (txout, keychain) = b.downcast_ref().cloned().unwrap();
                 Ok(Some(UTXO {
-                    outpoint: outpoint.clone(),
+                    outpoint: *outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 }))
             }
         }
@@ -272,8 +272,8 @@ impl BatchOperations for MemoryDatabase {
             }
         }
     }
-    fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
@@ -287,10 +287,10 @@ impl BatchOperations for MemoryDatabase {
 impl Database for MemoryDatabase {
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error> {
-        let key = MapKey::DescriptorChecksum(script_type).as_map_key();
+        let key = MapKey::DescriptorChecksum(keychain).as_map_key();
 
         let prev = self
             .map
@@ -308,8 +308,8 @@ impl Database for MemoryDatabase {
         }
     }
 
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error> {
-        let key = MapKey::Path((script_type, None)).as_map_key();
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        let key = MapKey::Path((keychain, None)).as_map_key();
         self.map
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(_, v)| Ok(v.downcast_ref().cloned().unwrap()))
@@ -322,11 +322,11 @@ impl Database for MemoryDatabase {
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(k, v)| {
                 let outpoint = deserialize(&k[1..]).unwrap();
-                let (txout, is_internal) = v.downcast_ref().cloned().unwrap();
+                let (txout, keychain) = v.downcast_ref().cloned().unwrap();
                 Ok(UTXO {
                     outpoint,
                     txout,
-                    is_internal,
+                    keychain,
                 })
             })
             .collect()
@@ -358,10 +358,10 @@ impl Database for MemoryDatabase {
 
     fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         path: u32,
     ) -> Result<Option<Script>, Error> {
-        let key = MapKey::Path((Some(script_type), Some(path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         Ok(self
             .map
             .get(&key)
@@ -371,7 +371,7 @@ impl Database for MemoryDatabase {
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
             let mut val: serde_json::Value = b.downcast_ref().cloned().unwrap();
@@ -385,11 +385,11 @@ impl Database for MemoryDatabase {
     fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
         let key = MapKey::UTXO(Some(outpoint)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
-            let (txout, is_internal) = b.downcast_ref().cloned().unwrap();
+            let (txout, keychain) = b.downcast_ref().cloned().unwrap();
             UTXO {
-                outpoint: outpoint.clone(),
+                outpoint: *outpoint,
                 txout,
-                is_internal,
+                keychain,
             }
         }))
     }
@@ -414,19 +414,19 @@ impl Database for MemoryDatabase {
         }))
     }
 
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         Ok(self.map.get(&key).map(|b| *b.downcast_ref().unwrap()))
     }
 
     // inserts 0 if not present
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         let value = self
             .map
-            .entry(key.clone())
+            .entry(key)
             .and_modify(|x| *x.downcast_mut::<u32>().unwrap() += 1)
-            .or_insert(Box::<u32>::new(0))
+            .or_insert_with(|| Box::<u32>::new(0))
             .downcast_mut()
             .unwrap();
 
@@ -445,8 +445,16 @@ impl BatchDatabase for MemoryDatabase {
         for key in batch.deleted_keys {
             self.map.remove(&key);
         }
+        self.map.append(&mut batch.map);
+        Ok(())
+    }
+}
 
-        Ok(self.map.append(&mut batch.map))
+impl ConfigurableDatabase for MemoryDatabase {
+    type Config = ();
+
+    fn from_config(_config: &Self::Config) -> Result<Self, Error> {
+        Ok(MemoryDatabase::default())
     }
 }
 
@@ -499,7 +507,7 @@ impl MemoryDatabase {
                     txid,
                     vout: vout as u32,
                 },
-                is_internal: false,
+                keychain: KeychainKind::External,
             })
             .unwrap();
         }

src/database/mod.rs

@@ -43,6 +43,9 @@ use bitcoin::{OutPoint, Script, Transaction, TxOut};
 use crate::error::Error;
 use crate::types::*;
 
+pub mod any;
+pub use any::{AnyDatabase, AnyDatabaseConfig};
+
 #[cfg(feature = "key-value-db")]
 pub(crate) mod keyvalue;
 
@@ -54,11 +57,11 @@ pub use memory::MemoryDatabase;
 /// This trait defines the list of operations that must be implemented on the [`Database`] type and
 /// the [`BatchDatabase::Batch`] type.
 pub trait BatchOperations {
-    /// Store a script_pubkey along with its script type and child number
+    /// Store a script_pubkey along with its keychain and child number.
     fn set_script_pubkey(
         &mut self,
         script: &Script,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         child: u32,
     ) -> Result<(), Error>;
     /// Store a [`UTXO`]
@@ -67,21 +70,21 @@ pub trait BatchOperations {
     fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error>;
     /// Store the metadata of a transaction
     fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error>;
-    /// Store the last derivation index for a given script type
-    fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error>;
+    /// Store the last derivation index for a given keychain.
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error>;
 
-    /// Delete a script_pubkey given the script type and its child number
+    /// Delete a script_pubkey given the keychain and its child number.
     fn del_script_pubkey_from_path(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         child: u32,
     ) -> Result<Option<Script>, Error>;
-    /// Delete the data related to a specific script_pubkey, meaning the script type and the child
-    /// number
+    /// Delete the data related to a specific script_pubkey, meaning the keychain and the child
+    /// number.
     fn del_path_from_script_pubkey(
         &mut self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error>;
+    ) -> Result<Option<(KeychainKind, u32)>, Error>;
     /// Delete a [`UTXO`] given its [`OutPoint`]
     fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
     /// Delete a raw transaction given its [`Txid`]
@@ -92,27 +95,27 @@ pub trait BatchOperations {
         txid: &Txid,
         include_raw: bool,
     ) -> Result<Option<TransactionDetails>, Error>;
-    /// Delete the last derivation index for a script type
-    fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error>;
+    /// Delete the last derivation index for a keychain.
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error>;
 }
 
 /// Trait for reading data from a database
 ///
 /// This traits defines the operations that can be used to read data out of a database
 pub trait Database: BatchOperations {
-    /// Read and checks the descriptor checksum for a given script type
+    /// Read and checks the descriptor checksum for a given keychain.
     ///
     /// Should return [`Error::ChecksumMismatch`](crate::error::Error::ChecksumMismatch) if the
     /// checksum doesn't match. If there's no checksum in the database, simply store it for the
     /// next time.
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error>;
 
     /// Return the list of script_pubkeys
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error>;
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error>;
     /// Return the list of [`UTXO`]s
     fn iter_utxos(&self) -> Result<Vec<UTXO>, Error>;
     /// Return the list of raw transactions
@@ -120,30 +123,30 @@ pub trait Database: BatchOperations {
     /// Return the list of transactions metadata
     fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error>;
 
-    /// Fetch a script_pubkey given the script type and child number
+    /// Fetch a script_pubkey given the child number of a keychain.
     fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         child: u32,
     ) -> Result<Option<Script>, Error>;
-    /// Fetch the script type and child number of a given script_pubkey
+    /// Fetch the keychain and child number of a given script_pubkey
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, u32)>, Error>;
+    ) -> Result<Option<(KeychainKind, u32)>, Error>;
     /// Fetch a [`UTXO`] given its [`OutPoint`]
     fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
     /// Fetch a raw transaction given its [`Txid`]
     fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error>;
     /// Fetch the transaction metadata and optionally also the raw transaction
     fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error>;
-    /// Return the last defivation index for a script type
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error>;
+    /// Return the last defivation index for a keychain.
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error>;
 
-    /// Increment the last derivation index for a script type and returns it
+    /// Increment the last derivation index for a keychain and return it
     ///
     /// It should insert and return `0` if not present in the database
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error>;
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error>;
 }
 
 /// Trait for a database that supports batch operations
@@ -159,6 +162,15 @@ pub trait BatchDatabase: Database {
     fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error>;
 }
 
+/// Trait for [`Database`] types that can be created given a configuration
+pub trait ConfigurableDatabase: Database + Sized {
+    /// Type that contains the configuration
+    type Config: std::fmt::Debug;
+
+    /// Create a new instance given a configuration
+    fn from_config(config: &Self::Config) -> Result<Self, Error>;
+}
+
 pub(crate) trait DatabaseUtils: Database {
     fn is_mine(&self, script: &Script) -> Result<bool, Error> {
         self.get_path_from_script_pubkey(script)
@@ -179,7 +191,7 @@ pub(crate) trait DatabaseUtils: Database {
         self.get_raw_tx(&outpoint.txid)?
             .map(|previous_tx| {
                 if outpoint.vout as usize >= previous_tx.output.len() {
-                    Err(Error::InvalidOutpoint(outpoint.clone()))
+                    Err(Error::InvalidOutpoint(*outpoint))
                 } else {
                     Ok(previous_tx.output[outpoint.vout as usize].clone())
                 }
@@ -205,17 +217,17 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        tree.set_script_pubkey(&script, script_type, path).unwrap();
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
 
         assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, path).unwrap(),
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
             Some(script.clone())
         );
         assert_eq!(
             tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
+            Some((keychain, path.clone()))
         );
     }
 
@@ -226,12 +238,12 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        batch.set_script_pubkey(&script, script_type, path).unwrap();
+        batch.set_script_pubkey(&script, keychain, path).unwrap();
 
         assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, path).unwrap(),
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
             None
         );
         assert_eq!(tree.get_path_from_script_pubkey(&script).unwrap(), None);
@@ -239,12 +251,12 @@ pub mod test {
         tree.commit_batch(batch).unwrap();
 
         assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, path).unwrap(),
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
             Some(script.clone())
         );
         assert_eq!(
             tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
+            Some((keychain, path.clone()))
         );
     }
 
@@ -253,9 +265,9 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        tree.set_script_pubkey(&script, script_type, path).unwrap();
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
 
         assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
     }
@@ -265,12 +277,12 @@ pub mod test {
             Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
         );
         let path = 42;
-        let script_type = ScriptType::External;
+        let keychain = KeychainKind::External;
 
-        tree.set_script_pubkey(&script, script_type, path).unwrap();
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
         assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
 
-        tree.del_script_pubkey_from_path(script_type, path).unwrap();
+        tree.del_script_pubkey_from_path(keychain, path).unwrap();
         assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0);
     }
 
@@ -289,7 +301,7 @@ pub mod test {
         let utxo = UTXO {
             txout,
             outpoint,
-            is_internal: false,
+            keychain: KeychainKind::External,
         };
 
         tree.set_utxo(&utxo).unwrap();
@@ -344,24 +356,27 @@ pub mod test {
     }
 
     pub fn test_last_index<D: Database>(mut tree: D) {
-        tree.set_last_index(ScriptType::External, 1337).unwrap();
+        tree.set_last_index(KeychainKind::External, 1337).unwrap();
 
         assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
+            tree.get_last_index(KeychainKind::External).unwrap(),
             Some(1337)
         );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), None);
+        assert_eq!(tree.get_last_index(KeychainKind::Internal).unwrap(), None);
 
-        let res = tree.increment_last_index(ScriptType::External).unwrap();
+        let res = tree.increment_last_index(KeychainKind::External).unwrap();
         assert_eq!(res, 1338);
-        let res = tree.increment_last_index(ScriptType::Internal).unwrap();
+        let res = tree.increment_last_index(KeychainKind::Internal).unwrap();
         assert_eq!(res, 0);
 
         assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
+            tree.get_last_index(KeychainKind::External).unwrap(),
             Some(1338)
         );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), Some(0));
+        assert_eq!(
+            tree.get_last_index(KeychainKind::Internal).unwrap(),
+            Some(0)
+        );
     }
 
     // TODO: more tests...

src/descriptor/checksum.rs

@@ -92,3 +92,35 @@ pub fn get_checksum(desc: &str) -> Result<String, Error> {
 
     Ok(String::from_iter(chars))
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::descriptor::get_checksum;
+
+    // test get_checksum() function; it should return the same value as Bitcoin Core
+    #[test]
+    fn test_get_checksum() {
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)";
+        assert_eq!(get_checksum(desc).unwrap(), "tqz0nc62");
+
+        let desc = "pkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/44'/1'/0'/0/*)";
+        assert_eq!(get_checksum(desc).unwrap(), "lasegmfs");
+    }
+
+    #[test]
+    fn test_get_checksum_invalid_character() {
+        let sparkle_heart = vec![240, 159, 146, 150];
+        let sparkle_heart = std::str::from_utf8(&sparkle_heart)
+            .unwrap()
+            .chars()
+            .next()
+            .unwrap();
+        let invalid_desc = format!("wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcL{}fjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)", sparkle_heart);
+
+        assert!(matches!(
+            get_checksum(&invalid_desc).err(),
+            Some(Error::InvalidDescriptorCharacter(invalid_char)) if invalid_char == sparkle_heart
+        ));
+    }
+}

src/descriptor/dsl.rs

@@ -0,0 +1,981 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Descriptors DSL
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_top_level_sh {
+    // disallow `sortedmulti` in `bare()`
+    ( Bare, Bare, sortedmulti $( $inner:tt )* ) => {
+        compile_error!("`bare()` descriptors can't contain any `sortedmulti()` operands");
+    };
+    ( Bare, Bare, sortedmulti_vec $( $inner:tt )* ) => {
+        compile_error!("`bare()` descriptors can't contain any `sortedmulti_vec()` operands");
+    };
+
+    ( $descriptor_variant:ident, $sortedmulti_variant:ident, sortedmulti $( $inner:tt )* ) => {
+        $crate::impl_sortedmulti!(sortedmulti $( $inner )*)
+            .and_then(|(inner, key_map, valid_networks)| Ok(($crate::miniscript::Descriptor::$sortedmulti_variant(inner), key_map, valid_networks)))
+    };
+    ( $descriptor_variant:ident, $sortedmulti_variant:ident, sortedmulti_vec $( $inner:tt )* ) => {
+        $crate::impl_sortedmulti!(sortedmulti_vec $( $inner )*)
+            .and_then(|(inner, key_map, valid_networks)| Ok(($crate::miniscript::Descriptor::$sortedmulti_variant(inner), key_map, valid_networks)))
+    };
+
+    ( $descriptor_variant:ident, $sortedmulti_variant:ident, $( $minisc:tt )* ) => {
+        $crate::fragment!($( $minisc )*)
+            .map(|(minisc, keymap, networks)|($crate::miniscript::Descriptor::<$crate::miniscript::descriptor::DescriptorPublicKey>::$descriptor_variant(minisc), keymap, networks))
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_top_level_pk {
+    ( $descriptor_variant:ident, $ctx:ty, $key:expr ) => {{
+        #[allow(unused_imports)]
+        use $crate::keys::{DescriptorKey, ToDescriptorKey};
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+
+        $key.to_descriptor_key()
+            .and_then(|key: DescriptorKey<$ctx>| key.extract(&secp))
+            .map(|(pk, key_map, valid_networks)| {
+                (
+                    $crate::miniscript::Descriptor::<
+                        $crate::miniscript::descriptor::DescriptorPublicKey,
+                    >::$descriptor_variant(pk),
+                    key_map,
+                    valid_networks,
+                )
+            })
+    }};
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_leaf_opcode {
+    ( $terminal_variant:ident ) => {
+        $crate::miniscript::Miniscript::from_ast(
+            $crate::miniscript::miniscript::decode::Terminal::$terminal_variant,
+        )
+        .map_err($crate::Error::Miniscript)
+        .map(|minisc| {
+            (
+                minisc,
+                $crate::miniscript::descriptor::KeyMap::default(),
+                $crate::keys::any_network(),
+            )
+        })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_leaf_opcode_value {
+    ( $terminal_variant:ident, $value:expr ) => {
+        $crate::miniscript::Miniscript::from_ast(
+            $crate::miniscript::miniscript::decode::Terminal::$terminal_variant($value),
+        )
+        .map_err($crate::Error::Miniscript)
+        .map(|minisc| {
+            (
+                minisc,
+                $crate::miniscript::descriptor::KeyMap::default(),
+                $crate::keys::any_network(),
+            )
+        })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_leaf_opcode_value_two {
+    ( $terminal_variant:ident, $one:expr, $two:expr ) => {
+        $crate::miniscript::Miniscript::from_ast(
+            $crate::miniscript::miniscript::decode::Terminal::$terminal_variant($one, $two),
+        )
+        .map_err($crate::Error::Miniscript)
+        .map(|minisc| {
+            (
+                minisc,
+                $crate::miniscript::descriptor::KeyMap::default(),
+                $crate::keys::any_network(),
+            )
+        })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_node_opcode_two {
+    ( $terminal_variant:ident, $( $inner:tt )* ) => ({
+        let inner = $crate::fragment_internal!( @t $( $inner )* );
+        let (a, b) = $crate::descriptor::dsl::TupleTwo::from(inner).flattened();
+
+        a
+            .and_then(|a| Ok((a, b?)))
+            .and_then(|((a_minisc, mut a_keymap, a_networks), (b_minisc, b_keymap, b_networks))| {
+                // join key_maps
+                a_keymap.extend(b_keymap.into_iter());
+
+                Ok(($crate::miniscript::Miniscript::from_ast($crate::miniscript::miniscript::decode::Terminal::$terminal_variant(
+                    std::sync::Arc::new(a_minisc),
+                    std::sync::Arc::new(b_minisc),
+                ))?, a_keymap, $crate::keys::merge_networks(&a_networks, &b_networks)))
+            })
+    });
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_node_opcode_three {
+    ( $terminal_variant:ident, $( $inner:tt )* ) => {
+        let inner = $crate::fragment_internal!( @t $( $inner )* );
+        let (a, b, c) = $crate::descriptor::dsl::TupleThree::from(inner).flattened();
+
+        a
+            .and_then(|a| Ok((a, b?, c?)))
+            .and_then(|((a_minisc, mut a_keymap, a_networks), (b_minisc, b_keymap, b_networks), (c_minisc, c_keymap, c_networks))| {
+                // join key_maps
+                a_keymap.extend(b_keymap.into_iter());
+                a_keymap.extend(c_keymap.into_iter());
+
+                let networks = $crate::keys::merge_networks(&a_networks, &b_networks);
+                let networks = $crate::keys::merge_networks(&networks, &c_networks);
+
+                Ok(($crate::miniscript::Miniscript::from_ast($crate::miniscript::miniscript::decode::Terminal::$terminal_variant(
+                    std::sync::Arc::new(a_minisc),
+                    std::sync::Arc::new(b_minisc),
+                    std::sync::Arc::new(c_minisc),
+                ))?, a_keymap, networks))
+            })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_sortedmulti {
+    ( sortedmulti_vec ( $thresh:expr, $keys:expr ) ) => ({
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+        $crate::keys::make_sortedmulti_inner($thresh, $keys, &secp)
+    });
+    ( sortedmulti ( $thresh:expr $(, $key:expr )+ ) ) => ({
+        use $crate::keys::ToDescriptorKey;
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+
+        let mut keys = vec![];
+        $(
+            keys.push($key.to_descriptor_key());
+        )*
+
+        keys.into_iter().collect::<Result<Vec<_>, _>>()
+            .and_then(|keys| $crate::keys::make_sortedmulti_inner($thresh, keys, &secp))
+    });
+
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! apply_modifier {
+    ( $terminal_variant:ident, $inner:expr ) => {{
+        $inner
+            .map_err(|e| -> $crate::Error { e.into() })
+            .and_then(|(minisc, keymap, networks)| {
+                Ok((
+                    $crate::miniscript::Miniscript::from_ast(
+                        $crate::miniscript::miniscript::decode::Terminal::$terminal_variant(
+                            std::sync::Arc::new(minisc),
+                        ),
+                    )?,
+                    keymap,
+                    networks,
+                ))
+            })
+    }};
+
+    ( a: $inner:expr ) => {{
+        $crate::apply_modifier!(Alt, $inner)
+    }};
+    ( s: $inner:expr ) => {{
+        $crate::apply_modifier!(Swap, $inner)
+    }};
+    ( c: $inner:expr ) => {{
+        $crate::apply_modifier!(Check, $inner)
+    }};
+    ( d: $inner:expr ) => {{
+        $crate::apply_modifier!(DupIf, $inner)
+    }};
+    ( v: $inner:expr ) => {{
+        $crate::apply_modifier!(Verify, $inner)
+    }};
+    ( j: $inner:expr ) => {{
+        $crate::apply_modifier!(NonZero, $inner)
+    }};
+    ( n: $inner:expr ) => {{
+        $crate::apply_modifier!(ZeroNotEqual, $inner)
+    }};
+
+    // Modifiers expanded to other operators
+    ( t: $inner:expr ) => {{
+        $inner.and_then(|(a_minisc, a_keymap, a_networks)| {
+            $crate::impl_leaf_opcode_value_two!(
+                AndV,
+                std::sync::Arc::new(a_minisc),
+                std::sync::Arc::new($crate::fragment!(true).unwrap().0)
+            )
+            .map(|(minisc, _, _)| (minisc, a_keymap, a_networks))
+        })
+    }};
+    ( l: $inner:expr ) => {{
+        $inner.and_then(|(a_minisc, a_keymap, a_networks)| {
+            $crate::impl_leaf_opcode_value_two!(
+                OrI,
+                std::sync::Arc::new($crate::fragment!(false).unwrap().0),
+                std::sync::Arc::new(a_minisc)
+            )
+            .map(|(minisc, _, _)| (minisc, a_keymap, a_networks))
+        })
+    }};
+    ( u: $inner:expr ) => {{
+        $inner.and_then(|(a_minisc, a_keymap, a_networks)| {
+            $crate::impl_leaf_opcode_value_two!(
+                OrI,
+                std::sync::Arc::new(a_minisc),
+                std::sync::Arc::new($crate::fragment!(false).unwrap().0)
+            )
+            .map(|(minisc, _, _)| (minisc, a_keymap, a_networks))
+        })
+    }};
+}
+
+/// Macro to write full descriptors with code
+///
+/// This macro expands to a `Result` of
+/// [`DescriptorTemplateOut`](super::template::DescriptorTemplateOut) and [`Error`](crate::Error)
+///
+/// The syntax is very similar to the normal descriptor syntax, with the exception that modifiers
+/// cannot be grouped together. For instance, a descriptor fragment like `sdv:older(144)` has to be
+/// broken up to `s:d:v:older(144)`.
+///
+/// The `pk()`, `pk_k()` and `pk_h()` operands can take as argument any type that implements
+/// [`ToDescriptorKey`]. This means that keys can also be written inline as strings, but in that
+/// case they must be wrapped in quotes, which is another difference compared to the standard
+/// descriptor syntax.
+///
+/// [`ToDescriptorKey`]: crate::keys::ToDescriptorKey
+///
+/// ## Example
+///
+/// Signature plus timelock descriptor:
+///
+/// ```
+/// # use std::str::FromStr;
+/// let (my_descriptor, my_keys_map, networks) = bdk::descriptor!(sh(wsh(and_v(v:pk("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy"),older(50)))))?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+///
+/// -------
+///
+/// 2-of-3 that becomes a 1-of-3 after a timelock has expired. Both `descriptor_a` and `descriptor_b` are equivalent: the first
+/// syntax is more suitable for a fixed number of items known at compile time, while the other accepts a
+/// [`Vec`] of items, which makes it more suitable for writing dynamic descriptors.
+///
+/// They both produce the descriptor: `wsh(thresh(2,pk(...),s:pk(...),sdv:older(...)))`
+///
+/// ```
+/// # use std::str::FromStr;
+/// let my_key_1 = bitcoin::PublicKey::from_str("02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c")?;
+/// let my_key_2 = bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
+/// let my_timelock = 50;
+///
+/// let (descriptor_a, key_map_a, networks) = bdk::descriptor! {
+///     wsh (
+///         thresh(2, pk(my_key_1), s:pk(my_key_2), s:d:v:older(my_timelock))
+///     )
+/// }?;
+///
+/// let b_items = vec![
+///     bdk::fragment!(pk(my_key_1))?,
+///     bdk::fragment!(s:pk(my_key_2))?,
+///     bdk::fragment!(s:d:v:older(my_timelock))?,
+/// ];
+/// let (descriptor_b, mut key_map_b, networks) = bdk::descriptor!(wsh(thresh_vec(2,b_items)))?;
+///
+/// assert_eq!(descriptor_a, descriptor_b);
+/// assert_eq!(key_map_a.len(), key_map_b.len());
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+///
+/// ------
+///
+/// Simple 2-of-2 multi-signature, equivalent to: `wsh(multi(2, ...))`
+///
+/// ```
+/// # use std::str::FromStr;
+/// let my_key_1 = bitcoin::PublicKey::from_str(
+///     "02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c",
+/// )?;
+/// let my_key_2 =
+///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
+///
+/// let (descriptor, key_map, networks) = bdk::descriptor! {
+///     wsh (
+///         multi(2, my_key_1, my_key_2)
+///     )
+/// }?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+///
+/// ------
+///
+/// Native-Segwit single-sig, equivalent to: `wpkh(...)`
+///
+/// ```
+/// let my_key =
+///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
+///
+/// let (descriptor, key_map, networks) = bdk::descriptor!(wpkh(my_key))?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+#[macro_export]
+macro_rules! descriptor {
+    ( bare ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(Bare, Bare, $( $minisc )*)
+    });
+    ( sh ( wsh ( $( $minisc:tt )* ) ) ) => ({
+        $crate::descriptor!(shwsh ($( $minisc )*))
+    });
+    ( shwsh ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(ShWsh, ShWshSortedMulti, $( $minisc )*)
+    });
+    ( pk ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(Pk, $crate::miniscript::Legacy, $key)
+    });
+    ( pkh ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(Pkh,$crate::miniscript::Legacy, $key)
+    });
+    ( wpkh ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(Wpkh, $crate::miniscript::Segwitv0, $key)
+    });
+    ( sh ( wpkh ( $key:expr ) ) ) => ({
+        $crate::descriptor!(shwpkh ( $key ))
+    });
+    ( shwpkh ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(ShWpkh, $crate::miniscript::Segwitv0, $key)
+    });
+    ( sh ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(Sh, ShSortedMulti, $( $minisc )*)
+    });
+    ( wsh ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(Wsh, WshSortedMulti, $( $minisc )*)
+    });
+}
+
+#[doc(hidden)]
+pub struct TupleTwo<A, B> {
+    pub a: A,
+    pub b: B,
+}
+
+impl<A, B> TupleTwo<A, B> {
+    pub fn flattened(self) -> (A, B) {
+        (self.a, self.b)
+    }
+}
+
+impl<A, B> From<(A, (B, ()))> for TupleTwo<A, B> {
+    fn from((a, (b, _)): (A, (B, ()))) -> Self {
+        TupleTwo { a, b }
+    }
+}
+
+#[doc(hidden)]
+pub struct TupleThree<A, B, C> {
+    pub a: A,
+    pub b: B,
+    pub c: C,
+}
+
+impl<A, B, C> TupleThree<A, B, C> {
+    pub fn flattened(self) -> (A, B, C) {
+        (self.a, self.b, self.c)
+    }
+}
+
+impl<A, B, C> From<(A, (B, (C, ())))> for TupleThree<A, B, C> {
+    fn from((a, (b, (c, _))): (A, (B, (C, ())))) -> Self {
+        TupleThree { a, b, c }
+    }
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! fragment_internal {
+    // The @v prefix is used to parse a sequence of operands and return them in a vector. This is
+    // used by operands that take a variable number of arguments, like `thresh()` and `multi()`.
+    ( @v $op:ident ( $( $args:tt )* ) $( $tail:tt )* ) => ({
+        let mut v = vec![$crate::fragment!( $op ( $( $args )* ) )];
+        v.append(&mut $crate::fragment_internal!( @v $( $tail )* ));
+
+        v
+    });
+    // Match modifiers
+    ( @v $modif:tt : $( $tail:tt )* ) => ({
+        let mut v = $crate::fragment_internal!( @v $( $tail )* );
+        let first = v.drain(..1).next().unwrap();
+
+        let first = $crate::apply_modifier!($modif:first);
+
+        let mut v_final = vec![first];
+        v_final.append(&mut v);
+
+        v_final
+    });
+    // Remove commas between operands
+    ( @v , $( $tail:tt )* ) => ({
+        $crate::fragment_internal!( @v $( $tail )* )
+    });
+    ( @v ) => ({
+        vec![]
+    });
+
+    // The @t prefix is used to parse a sequence of operands and return them in a tuple. This
+    // allows checking at compile-time the number of arguments passed to an operand. For this
+    // reason it's used by `and_*()`, `or_*()`, etc.
+    //
+    // Unfortunately, due to the fact that concatenating tuples is pretty hard, the final result
+    // adds in the first spot the parsed operand and in the second spot the result of parsing
+    // all the following ones. For two operands the type then corresponds to: (X, (X, ())). For
+    // three operands it's (X, (X, (X, ()))), etc.
+    //
+    // To check that the right number of arguments has been passed we can "cast" those tuples to
+    // more convenient structures like `TupleTwo`. If the conversion succedes, the right number of
+    // args was passed. Otherwise the compilation fails entirely.
+    ( @t $op:ident ( $( $args:tt )* ) $( $tail:tt )* ) => ({
+        ($crate::fragment!( $op ( $( $args )* ) ), $crate::fragment_internal!( @t $( $tail )* ))
+    });
+    // Match modifiers
+    ( @t $modif:tt : $( $tail:tt )* ) => ({
+        let (first, tail) = $crate::fragment_internal!( @t $( $tail )* );
+        ($crate::apply_modifier!($modif:first), tail)
+    });
+    // Remove commas between operands
+    ( @t , $( $tail:tt )* ) => ({
+        $crate::fragment_internal!( @t $( $tail )* )
+    });
+    ( @t ) => ({
+        ()
+    });
+
+    // Fallback to calling `fragment!()`
+    ( $( $tokens:tt )* ) => ({
+        $crate::fragment!($( $tokens )*)
+    });
+}
+
+/// Macro to write descriptor fragments with code
+///
+/// This macro will be expanded to an object of type `Result<(Miniscript<DescriptorPublicKey, _>, KeyMap, ValidNetworks), Error>`. It allows writing
+/// fragments of larger descriptors that can be pieced together using `fragment!(thresh_vec(m, ...))`.
+///
+/// The syntax to write macro fragment is the same as documented for the [`descriptor`] macro.
+#[macro_export]
+macro_rules! fragment {
+    // Modifiers
+    ( $modif:tt : $( $tail:tt )* ) => ({
+        let op = $crate::fragment!( $( $tail )* );
+        $crate::apply_modifier!($modif:op)
+    });
+
+    // Miniscript
+    ( true ) => ({
+        $crate::impl_leaf_opcode!(True)
+    });
+    ( false ) => ({
+        $crate::impl_leaf_opcode!(False)
+    });
+    ( pk_k ( $key:expr ) ) => ({
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+        $crate::keys::make_pk($key, &secp)
+    });
+    ( pk ( $key:expr ) ) => ({
+        $crate::fragment!(c:pk_k ( $key ))
+    });
+    ( pk_h ( $key_hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(PkH, $key_hash)
+    });
+    ( after ( $value:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(After, $value)
+    });
+    ( older ( $value:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Older, $value)
+    });
+    ( sha256 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Sha256, $hash)
+    });
+    ( hash256 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Hash256, $hash)
+    });
+    ( ripemd160 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Ripemd160, $hash)
+    });
+    ( hash160 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Hash160, $hash)
+    });
+    ( and_v ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(AndV, $( $inner )*)
+    });
+    ( and_b ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(AndB, $( $inner )*)
+    });
+    ( and_or ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_three!(AndOr, $( $inner )*)
+    });
+    ( or_b ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrB, $( $inner )*)
+    });
+    ( or_d ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrD, $( $inner )*)
+    });
+    ( or_c ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrC, $( $inner )*)
+    });
+    ( or_i ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrI, $( $inner )*)
+    });
+    ( thresh_vec ( $thresh:expr, $items:expr ) ) => ({
+        use $crate::miniscript::descriptor::KeyMap;
+
+        let (items, key_maps_networks): (Vec<_>, Vec<_>) = $items.into_iter().map(|(a, b, c)| (a, (b, c))).unzip();
+        let items = items.into_iter().map(std::sync::Arc::new).collect();
+
+        let (key_maps, valid_networks) = key_maps_networks.into_iter().fold((KeyMap::default(), $crate::keys::any_network()), |(mut keys_acc, net_acc), (key, net)| {
+            keys_acc.extend(key.into_iter());
+            let net_acc = $crate::keys::merge_networks(&net_acc, &net);
+
+            (keys_acc, net_acc)
+        });
+
+        $crate::impl_leaf_opcode_value_two!(Thresh, $thresh, items)
+            .map(|(minisc, _, _)| (minisc, key_maps, valid_networks))
+    });
+    ( thresh ( $thresh:expr, $( $inner:tt )* ) ) => ({
+        let items = $crate::fragment_internal!( @v $( $inner )* );
+
+        items.into_iter().collect::<Result<Vec<_>, _>>()
+            .and_then(|items| $crate::fragment!(thresh_vec($thresh, items)))
+    });
+    ( multi_vec ( $thresh:expr, $keys:expr ) ) => ({
+        $crate::keys::make_multi($thresh, $keys)
+    });
+    ( multi ( $thresh:expr $(, $key:expr )+ ) ) => ({
+        use $crate::keys::ToDescriptorKey;
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+
+        let mut keys = vec![];
+        $(
+            keys.push($key.to_descriptor_key());
+        )*
+
+        keys.into_iter().collect::<Result<Vec<_>, _>>()
+            .and_then(|keys| $crate::keys::make_multi($thresh, keys, &secp))
+    });
+
+    // `sortedmulti()` is handled separately
+    ( sortedmulti ( $( $inner:tt )* ) ) => ({
+        compile_error!("`sortedmulti` can only be used as the root operand of a descriptor");
+    });
+    ( sortedmulti_vec ( $( $inner:tt )* ) ) => ({
+        compile_error!("`sortedmulti_vec` can only be used as the root operand of a descriptor");
+    });
+}
+
+#[cfg(test)]
+mod test {
+    use bitcoin::hashes::hex::ToHex;
+    use bitcoin::secp256k1::Secp256k1;
+    use miniscript::descriptor::{DescriptorPublicKey, DescriptorPublicKeyCtx, KeyMap};
+    use miniscript::{Descriptor, Legacy, Segwitv0};
+
+    use std::str::FromStr;
+
+    use crate::descriptor::DescriptorMeta;
+    use crate::keys::{DescriptorKey, KeyError, ToDescriptorKey, ValidNetworks};
+    use bitcoin::network::constants::Network::{Bitcoin, Regtest, Testnet};
+    use bitcoin::util::bip32;
+    use bitcoin::util::bip32::ChildNumber;
+    use bitcoin::PrivateKey;
+
+    // test the descriptor!() macro
+
+    // verify descriptor generates expected script(s) (if bare or pk) or address(es)
+    fn check(
+        desc: Result<(Descriptor<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError>,
+        is_witness: bool,
+        is_fixed: bool,
+        expected: &[&str],
+    ) {
+        let secp = Secp256k1::new();
+        let deriv_ctx = DescriptorPublicKeyCtx::new(&secp, ChildNumber::Normal { index: 0 });
+
+        let (desc, _key_map, _networks) = desc.unwrap();
+        assert_eq!(desc.is_witness(), is_witness);
+        assert_eq!(desc.is_fixed(), is_fixed);
+        for i in 0..expected.len() {
+            let index = i as u32;
+            let child_desc = if desc.is_fixed() {
+                desc.clone()
+            } else {
+                desc.derive(ChildNumber::from_normal_idx(index).unwrap())
+            };
+            let address = child_desc.address(Regtest, deriv_ctx);
+            if let Some(address) = address {
+                assert_eq!(address.to_string(), *expected.get(i).unwrap());
+            } else {
+                let script = child_desc.script_pubkey(deriv_ctx);
+                assert_eq!(script.to_hex().as_str(), *expected.get(i).unwrap());
+            }
+        }
+    }
+
+    // - at least one of each "type" of operator; ie. one modifier, one leaf_opcode, one leaf_opcode_value, etc.
+    // - mixing up key types that implement ToDescriptorKey in multi() or thresh()
+
+    // expected script for pk and bare manually created
+    // expected addresses created with `bitcoin-cli getdescriptorinfo` (for hash) and `bitcoin-cli deriveaddresses`
+
+    #[test]
+    fn test_fixed_legacy_descriptors() {
+        let pubkey1 = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        let pubkey2 = bitcoin::PublicKey::from_str(
+            "032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af",
+        )
+        .unwrap();
+
+        check(
+            descriptor!(bare(multi(1,pubkey1,pubkey2))),
+            false,
+            true,
+            &["512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd21032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af52ae"],
+        );
+        check(
+            descriptor!(pk(pubkey1)),
+            false,
+            true,
+            &["2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac"],
+        );
+        check(
+            descriptor!(pkh(pubkey1)),
+            false,
+            true,
+            &["muZpTpBYhxmRFuCjLc7C6BBDF32C8XVJUi"],
+        );
+        check(
+            descriptor!(sh(multi(1, pubkey1, pubkey2))),
+            false,
+            true,
+            &["2MymURoV1bzuMnWMGiXzyomDkeuxXY7Suey"],
+        );
+    }
+
+    #[test]
+    fn test_fixed_segwitv0_descriptors() {
+        let pubkey1 = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        let pubkey2 = bitcoin::PublicKey::from_str(
+            "032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af",
+        )
+        .unwrap();
+
+        check(
+            descriptor!(wpkh(pubkey1)),
+            true,
+            true,
+            &["bcrt1qngw83fg8dz0k749cg7k3emc7v98wy0c7azaa6h"],
+        );
+        check(
+            descriptor!(sh(wpkh(pubkey1))),
+            true,
+            true,
+            &["2N5LiC3CqzxDamRTPG1kiNv1FpNJQ7x28sb"],
+        );
+        check(
+            descriptor!(wsh(multi(1, pubkey1, pubkey2))),
+            true,
+            true,
+            &["bcrt1qgw8jvv2hsrvjfa6q66rk6har7d32lrqm5unnf5cl63q9phxfvgps5fyfqe"],
+        );
+        check(
+            descriptor!(sh(wsh(multi(1, pubkey1, pubkey2)))),
+            true,
+            true,
+            &["2NCidRJysy7apkmE6JF5mLLaJFkrN3Ub9iy"],
+        );
+    }
+
+    #[test]
+    fn test_bip32_legacy_descriptors() {
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(pk(desc_key)),
+            false,
+            false,
+            &[
+                "2102363ad03c10024e1b597a5b01b9982807fb638e00b06f3b2d4a89707de3b93c37ac",
+                "2102063a21fd780df370ed2fc8c4b86aa5ea642630609c203009df631feb7b480dd2ac",
+                "2102ba2685ad1fa5891cb100f1656b2ce3801822ccb9bac0336734a6f8c1b93ebbc0ac",
+            ],
+        );
+
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(pkh(desc_key)),
+            false,
+            false,
+            &[
+                "muvBdsVpJxpFuTHMKA47htJPdCvdt4F9DP",
+                "mxQSHK7DL2t1DN3xFxov1janCoXSSkrSPj",
+                "mfz43r15GiWo4nizmyzMNubsnkDpByFFAn",
+            ],
+        );
+
+        let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
+        let desc_key1 = (xprv, path).to_descriptor_key().unwrap();
+        let desc_key2 = (xprv, path2).to_descriptor_key().unwrap();
+
+        check(
+            descriptor!(sh(multi(1, desc_key1, desc_key2))),
+            false,
+            false,
+            &[
+                "2MtMDXsfwefZkEEhVViEPidvcKRUtJamJJ8",
+                "2MwAUZ1NYyWjhVvGTethFL6n7nZhS8WE6At",
+                "2MuT6Bj66HLwZd7s4SoD8XbK4GwriKEA6Gr",
+            ],
+        );
+    }
+
+    #[test]
+    fn test_bip32_segwitv0_descriptors() {
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(wpkh(desc_key)),
+            true,
+            false,
+            &[
+                "bcrt1qnhm8w9fhc8cxzgqsmqdf9fyjccyvc0gltnymu0",
+                "bcrt1qhylfd55rn75w9fj06zspctad5w4hz33rf0ttad",
+                "bcrt1qq5sq3a6k9av9d8cne0k9wcldy4nqey5yt6889r",
+            ],
+        );
+
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(sh(wpkh(desc_key))),
+            true,
+            false,
+            &[
+                "2MxvjQCaLqZ5QxZ7XotZDQ63hZw3NPss763",
+                "2NDUoevN4QMzhvHDMGhKuiT2fN9HXbFRMwn",
+                "2NF4BEAY2jF1Fu8vqfN3NVKoFtom77pUxrx",
+            ],
+        );
+
+        let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
+        let desc_key1 = (xprv, path.clone()).to_descriptor_key().unwrap();
+        let desc_key2 = (xprv, path2.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(wsh(multi(1, desc_key1, desc_key2))),
+            true,
+            false,
+            &[
+                "bcrt1qfxv8mxmlv5sz8q2mnuyaqdfe9jr4vvmx0csjhn092p6f4qfygfkq2hng49",
+                "bcrt1qerj85g243e6jlcdxpmn9spk0gefcwvu7nw7ee059d5ydzpdhkm2qwfkf5k",
+                "bcrt1qxkl2qss3k58q9ktc8e89pwr4gnptfpw4hju4xstxcjc0hkcae3jstluty7",
+            ],
+        );
+
+        let desc_key1 = (xprv, path).to_descriptor_key().unwrap();
+        let desc_key2 = (xprv, path2).to_descriptor_key().unwrap();
+        check(
+            descriptor!(sh(wsh(multi(1, desc_key1, desc_key2)))),
+            true,
+            false,
+            &[
+                "2NFCtXvx9q4ci2kvKub17iSTgvRXGctCGhz",
+                "2NB2PrFPv5NxWCpygas8tPrGJG2ZFgeuwJw",
+                "2N79ZAGo5cMi5Jt7Wo9L5YmF5GkEw7sjWdC",
+            ],
+        );
+    }
+
+    #[test]
+    fn test_dsl_sortedmulti() {
+        let key_1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path_1 = bip32::DerivationPath::from_str("m/0").unwrap();
+
+        let key_2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let path_2 = bip32::DerivationPath::from_str("m/1").unwrap();
+
+        let desc_key1 = (key_1, path_1);
+        let desc_key2 = (key_2, path_2);
+
+        check(
+            descriptor!(sh(sortedmulti(1, desc_key1.clone(), desc_key2.clone()))),
+            false,
+            false,
+            &[
+                "2MsxzPEJDBzpGffJXPaDpfXZAUNnZhaMh2N",
+                "2My3x3DLPK3UbGWGpxrXr1RnbD8MNC4FpgS",
+                "2NByEuiQT7YLqHCTNxL5KwYjvtuCYcXNBSC",
+                "2N1TGbP81kj2VUKTSWgrwxoMfuWjvfUdyu7",
+                "2N3Bomq2fpAcLRNfZnD3bCWK9quan28CxCR",
+                "2N9nrZaEzEFDqEAU9RPvDnXGT6AVwBDKAQb",
+            ],
+        );
+
+        check(
+            descriptor!(sh(wsh(sortedmulti(
+                1,
+                desc_key1.clone(),
+                desc_key2.clone()
+            )))),
+            true,
+            false,
+            &[
+                "2NCogc5YyM4N6ruv1hUa7WLMW1BPeCK7N9B",
+                "2N6mkSAKi1V2oaBXby7XHdvBMKEDRQcFpNe",
+                "2NFmTSttm9v6bXeoWaBvpMcgfPQcZhNn3Eh",
+                "2Mvib87RBPUHXNEpX5S5Kv1qqrhBfgBGsJM",
+                "2MtMv5mcK2EjcLsH8Txpx2JxLLzHr4ttczL",
+                "2MsWCB56rb4T6yPv8QudZGHERTwNgesE4f6",
+            ],
+        );
+
+        check(
+            descriptor!(wsh(sortedmulti_vec(1, vec![desc_key1, desc_key2]))),
+            true,
+            false,
+            &[
+                "bcrt1qcvq0lg8q7a47ytrd7zk5y7uls7mulrenjgvflwylpppgwf8029es4vhpnj",
+                "bcrt1q80yn8sdt6l7pjvkz25lglyaqctlmsq9ugk80rmxt8yu0npdsj97sc7l4de",
+                "bcrt1qrvf6024v9s50qhffe3t2fr2q9ckdhx2g6jz32chm2pp24ymgtr5qfrdmct",
+                "bcrt1q6srfmra0ynypym35c7jvsxt2u4yrugeajq95kg2ps7lk6h2gaunsq9lzxn",
+                "bcrt1qhl8rrzzcdpu7tcup3lcg7tge52sqvwy5fcv4k78v6kxtwmqf3v6qpvyjza",
+                "bcrt1ql2elz9mhm9ll27ddpewhxs732xyl2fk2kpkqz9gdyh33wgcun4vstrd49k",
+            ],
+        );
+    }
+
+    // - verify the valid_networks returned is correctly computed based on the keys present in the descriptor
+    #[test]
+    fn test_valid_networks() {
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+
+        let (_desc, _key_map, valid_networks) = descriptor!(pkh(desc_key)).unwrap();
+        assert_eq!(valid_networks, [Testnet, Regtest].iter().cloned().collect());
+
+        let xprv = bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi").unwrap();
+        let path = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+
+        let (_desc, _key_map, valid_networks) = descriptor!(wpkh(desc_key)).unwrap();
+        assert_eq!(valid_networks, [Bitcoin].iter().cloned().collect());
+    }
+
+    // - verify the key_maps are correctly merged together
+    #[test]
+    fn test_key_maps_merged() {
+        let secp = Secp256k1::new();
+
+        let xprv1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path1 = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key1 = (xprv1, path1.clone()).to_descriptor_key().unwrap();
+
+        let xprv2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
+        let desc_key2 = (xprv2, path2.clone()).to_descriptor_key().unwrap();
+
+        let xprv3 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf").unwrap();
+        let path3 = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
+        let desc_key3 = (xprv3, path3.clone()).to_descriptor_key().unwrap();
+
+        let (_desc, key_map, _valid_networks) =
+            descriptor!(sh(wsh(multi(2, desc_key1, desc_key2, desc_key3)))).unwrap();
+        assert_eq!(key_map.len(), 3);
+
+        let desc_key1: DescriptorKey<Segwitv0> =
+            (xprv1, path1.clone()).to_descriptor_key().unwrap();
+        let desc_key2: DescriptorKey<Segwitv0> =
+            (xprv2, path2.clone()).to_descriptor_key().unwrap();
+        let desc_key3: DescriptorKey<Segwitv0> =
+            (xprv3, path3.clone()).to_descriptor_key().unwrap();
+
+        let (key1, _key_map, _valid_networks) = desc_key1.extract(&secp).unwrap();
+        let (key2, _key_map, _valid_networks) = desc_key2.extract(&secp).unwrap();
+        let (key3, _key_map, _valid_networks) = desc_key3.extract(&secp).unwrap();
+        assert_eq!(key_map.get(&key1).unwrap().to_string(), "tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy/0/*");
+        assert_eq!(key_map.get(&key2).unwrap().to_string(), "tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF/2147483647'/0/*");
+        assert_eq!(key_map.get(&key3).unwrap().to_string(), "tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf/10/20/30/40/*");
+    }
+
+    // - verify the ScriptContext is correctly validated (i.e. passing a type that only impl ToDescriptorKey<Segwitv0> to a pkh() descriptor should throw a compilation error
+    #[test]
+    fn test_script_context_validation() {
+        // this compiles
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key: DescriptorKey<Legacy> = (xprv, path.clone()).to_descriptor_key().unwrap();
+
+        let (desc, _key_map, _valid_networks) = descriptor!(pkh(desc_key)).unwrap();
+        assert_eq!(desc.to_string(), "pkh(tpubD6NzVbkrYhZ4WR7a4vY1VT3khMJMeAxVsfq9TBJyJWrNk247zCJtV7AWf6UJP7rAVsn8NNKdJi3gFyKPTmWZS9iukb91xbn2HbFSMQm2igY/0/*)");
+
+        // as expected this does not compile due to invalid context
+        //let desc_key:DescriptorKey<Segwitv0> = (xprv, path.clone()).to_descriptor_key().unwrap();
+        //let (desc, _key_map, _valid_networks) = descriptor!(pkh(desc_key)).unwrap();
+    }
+
+    #[test]
+    fn test_dsl_modifiers() {
+        let private_key =
+            PrivateKey::from_wif("cSQPHDBwXGjVzWRqAHm6zfvQhaTuj1f2bFH58h55ghbjtFwvmeXR").unwrap();
+        let (descriptor, _, _) =
+            descriptor!(wsh(thresh(2,d:v:older(1),s:pk(private_key),s:pk(private_key)))).unwrap();
+
+        assert_eq!(descriptor.to_string(), "wsh(thresh(2,dv:older(1),s:pk(02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c),s:pk(02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c)))")
+    }
+}

src/descriptor/error.rs

@@ -27,29 +27,48 @@
 /// Errors related to the parsing and usage of descriptors
 #[derive(Debug)]
 pub enum Error {
-    InternalError,
-    InvalidPrefix(Vec<u8>),
-    HardenedDerivationOnXpub,
-    MalformedInput,
-    KeyParsingError(String),
+    //InternalError,
+    //InvalidPrefix(Vec<u8>),
+    //HardenedDerivationOnXpub,
+    //MalformedInput,
+    /// Invalid HD Key path, such as having a wildcard but a length != 1
+    InvalidHDKeyPath,
 
+    //KeyParsingError(String),
+    /// Error thrown while working with [`keys`](crate::keys)
+    Key(crate::keys::KeyError),
+    /// Error while extracting and manipulating policies
     Policy(crate::descriptor::policy::PolicyError),
 
-    InputIndexDoesntExist,
-    MissingPublicKey,
-    MissingDetails,
-
+    //InputIndexDoesntExist,
+    //MissingPublicKey,
+    //MissingDetails,
+    /// Invalid character found in the descriptor checksum
     InvalidDescriptorCharacter(char),
 
-    CantDeriveWithMiniscript,
-
+    //CantDeriveWithMiniscript,
+    /// BIP32 error
     BIP32(bitcoin::util::bip32::Error),
+    /// Error during base58 decoding
     Base58(bitcoin::util::base58::Error),
+    /// Key-related error
     PK(bitcoin::util::key::Error),
+    /// Miniscript error
     Miniscript(miniscript::Error),
+    /// Hex decoding error
     Hex(bitcoin::hashes::hex::Error),
 }
 
+impl From<crate::keys::KeyError> for Error {
+    fn from(key_error: crate::keys::KeyError) -> Error {
+        match key_error {
+            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
+            crate::keys::KeyError::BIP32(inner) => Error::BIP32(inner),
+            e => Error::Key(e),
+        }
+    }
+}
+
 impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{:?}", self)

src/descriptor/mod.rs

@@ -29,27 +29,32 @@
 
 use std::collections::{BTreeMap, HashMap};
 use std::fmt;
-use std::sync::Arc;
 
-use bitcoin::hashes::hash160;
 use bitcoin::secp256k1::Secp256k1;
-use bitcoin::util::bip32::{ChildNumber, DerivationPath, Fingerprint};
+use bitcoin::util::bip32::{ChildNumber, DerivationPath, ExtendedPubKey, Fingerprint, KeySource};
 use bitcoin::util::psbt;
-use bitcoin::{PublicKey, Script, TxOut};
+use bitcoin::{Network, PublicKey, Script, TxOut};
 
 use miniscript::descriptor::{DescriptorPublicKey, DescriptorXKey, InnerXKey};
 pub use miniscript::{
-    Descriptor, Legacy, Miniscript, MiniscriptKey, ScriptContext, Segwitv0, Terminal, ToPublicKey,
+    descriptor::KeyMap, Descriptor, Legacy, Miniscript, MiniscriptKey, ScriptContext, Segwitv0,
+    Terminal, ToPublicKey,
 };
 
 pub mod checksum;
+#[doc(hidden)]
+pub mod dsl;
 pub mod error;
 pub mod policy;
+pub mod template;
 
 pub use self::checksum::get_checksum;
 use self::error::Error;
 pub use self::policy::Policy;
+use self::template::DescriptorTemplateOut;
+use crate::keys::{KeyError, ToDescriptorKey};
 use crate::wallet::signer::SignersContainer;
+use crate::wallet::utils::{descriptor_to_pk_ctx, SecpCtx};
 
 /// Alias for a [`Descriptor`] that can contain extended keys using [`DescriptorPublicKey`]
 pub type ExtendedDescriptor = Descriptor<DescriptorPublicKey>;
@@ -59,36 +64,160 @@ pub type ExtendedDescriptor = Descriptor<DescriptorPublicKey>;
 ///
 /// [`psbt::Input`]: bitcoin::util::psbt::Input
 /// [`psbt::Output`]: bitcoin::util::psbt::Output
-pub type HDKeyPaths = BTreeMap<PublicKey, (Fingerprint, DerivationPath)>;
+pub type HDKeyPaths = BTreeMap<PublicKey, KeySource>;
+
+/// Trait for types which can be converted into an [`ExtendedDescriptor`] and a [`KeyMap`] usable by a wallet in a specific [`Network`]
+pub trait ToWalletDescriptor {
+    /// Convert to wallet descriptor
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError>;
+}
+
+impl ToWalletDescriptor for &str {
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError> {
+        let descriptor = if self.contains('#') {
+            let parts: Vec<&str> = self.splitn(2, '#').collect();
+            if !get_checksum(parts[0])
+                .ok()
+                .map(|computed| computed == parts[1])
+                .unwrap_or(false)
+            {
+                return Err(KeyError::InvalidChecksum);
+            }
+
+            parts[0]
+        } else {
+            self
+        };
+
+        ExtendedDescriptor::parse_descriptor(descriptor)?.to_wallet_descriptor(network)
+    }
+}
+
+impl ToWalletDescriptor for &String {
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError> {
+        self.as_str().to_wallet_descriptor(network)
+    }
+}
+
+impl ToWalletDescriptor for ExtendedDescriptor {
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError> {
+        (self, KeyMap::default()).to_wallet_descriptor(network)
+    }
+}
+
+impl ToWalletDescriptor for (ExtendedDescriptor, KeyMap) {
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError> {
+        use crate::keys::DescriptorKey;
+
+        let secp = Secp256k1::new();
+
+        let check_key = |pk: &DescriptorPublicKey| {
+            let (pk, _, networks) = if self.0.is_witness() {
+                let desciptor_key: DescriptorKey<miniscript::Segwitv0> =
+                    pk.clone().to_descriptor_key()?;
+                desciptor_key.extract(&secp)?
+            } else {
+                let desciptor_key: DescriptorKey<miniscript::Legacy> =
+                    pk.clone().to_descriptor_key()?;
+                desciptor_key.extract(&secp)?
+            };
+
+            if networks.contains(&network) {
+                Ok(pk)
+            } else {
+                Err(KeyError::InvalidNetwork)
+            }
+        };
+
+        // check the network for the keys
+        let translated = self.0.translate_pk(check_key, check_key)?;
+
+        Ok((translated, self.1))
+    }
+}
+
+impl ToWalletDescriptor for DescriptorTemplateOut {
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError> {
+        let valid_networks = &self.2;
+
+        let fix_key = |pk: &DescriptorPublicKey| {
+            if valid_networks.contains(&network) {
+                // workaround for xpubs generated by other key types, like bip39: since when the
+                // conversion is made one network has to be chosen, what we generally choose
+                // "mainnet", but then override the set of valid networks to specify that all of
+                // them are valid. here we reset the network to make sure the wallet struct gets a
+                // descriptor with the right network everywhere.
+                let pk = match pk {
+                    DescriptorPublicKey::XPub(ref xpub) => {
+                        let mut xpub = xpub.clone();
+                        xpub.xkey.network = network;
+
+                        DescriptorPublicKey::XPub(xpub)
+                    }
+                    other => other.clone(),
+                };
+
+                Ok(pk)
+            } else {
+                Err(KeyError::InvalidNetwork)
+            }
+        };
+
+        // fixup the network for keys that need it
+        let translated = self.0.translate_pk(fix_key, fix_key)?;
+
+        Ok((translated, self.1))
+    }
+}
 
 /// Trait implemented on [`Descriptor`]s to add a method to extract the spending [`policy`]
 pub trait ExtractPolicy {
+    /// Extract the spending [`policy`]
     fn extract_policy(
         &self,
-        signers: Arc<SignersContainer<DescriptorPublicKey>>,
+        signers: &SignersContainer,
+        secp: &SecpCtx,
     ) -> Result<Option<Policy>, Error>;
 }
 
 pub(crate) trait XKeyUtils {
     fn full_path(&self, append: &[ChildNumber]) -> DerivationPath;
-    fn root_fingerprint(&self) -> Fingerprint;
+    fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint;
 }
 
 impl<K: InnerXKey> XKeyUtils for DescriptorXKey<K> {
     fn full_path(&self, append: &[ChildNumber]) -> DerivationPath {
-        let full_path = match &self.source {
-            &Some((_, ref path)) => path
+        let full_path = match self.origin {
+            Some((_, ref path)) => path
                 .into_iter()
                 .chain(self.derivation_path.into_iter())
                 .cloned()
                 .collect(),
-            &None => self.derivation_path.clone(),
+            None => self.derivation_path.clone(),
         };
 
         if self.is_wildcard {
             full_path
                 .into_iter()
-                .chain(append.into_iter())
+                .chain(append.iter())
                 .cloned()
                 .collect()
         } else {
@@ -96,46 +225,56 @@ impl<K: InnerXKey> XKeyUtils for DescriptorXKey<K> {
         }
     }
 
-    fn root_fingerprint(&self) -> Fingerprint {
-        match &self.source {
-            &Some((fingerprint, _)) => fingerprint.clone(),
-            &None => self.xkey.xkey_fingerprint(),
+    fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint {
+        match self.origin {
+            Some((fingerprint, _)) => fingerprint,
+            None => self.xkey.xkey_fingerprint(secp),
         }
     }
 }
 
 pub(crate) trait DescriptorMeta: Sized {
     fn is_witness(&self) -> bool;
-    fn get_hd_keypaths(&self, index: u32) -> Result<HDKeyPaths, Error>;
+    fn get_hd_keypaths(&self, index: u32, secp: &SecpCtx) -> Result<HDKeyPaths, Error>;
+    fn get_extended_keys(&self) -> Result<Vec<DescriptorXKey<ExtendedPubKey>>, Error>;
     fn is_fixed(&self) -> bool;
-    fn derive_from_hd_keypaths(&self, hd_keypaths: &HDKeyPaths) -> Option<Self>;
-    fn derive_from_psbt_input(&self, psbt_input: &psbt::Input, utxo: Option<TxOut>)
-        -> Option<Self>;
+    fn derive_from_hd_keypaths(&self, hd_keypaths: &HDKeyPaths, secp: &SecpCtx) -> Option<Self>;
+    fn derive_from_psbt_input(
+        &self,
+        psbt_input: &psbt::Input,
+        utxo: Option<TxOut>,
+        secp: &SecpCtx,
+    ) -> Option<Self>;
 }
 
 pub(crate) trait DescriptorScripts {
-    fn psbt_redeem_script(&self) -> Option<Script>;
-    fn psbt_witness_script(&self) -> Option<Script>;
+    fn psbt_redeem_script(&self, secp: &SecpCtx) -> Option<Script>;
+    fn psbt_witness_script(&self, secp: &SecpCtx) -> Option<Script>;
 }
 
-impl<T> DescriptorScripts for Descriptor<T>
-where
-    T: miniscript::MiniscriptKey + miniscript::ToPublicKey,
-{
-    fn psbt_redeem_script(&self) -> Option<Script> {
+impl DescriptorScripts for Descriptor<DescriptorPublicKey> {
+    fn psbt_redeem_script(&self, secp: &SecpCtx) -> Option<Script> {
+        let deriv_ctx = descriptor_to_pk_ctx(secp);
+
         match self {
-            Descriptor::ShWpkh(_) => Some(self.witness_script()),
-            Descriptor::ShWsh(ref script) => Some(script.encode().to_v0_p2wsh()),
-            Descriptor::Sh(ref script) => Some(script.encode()),
-            Descriptor::Bare(ref script) => Some(script.encode()),
+            Descriptor::ShWpkh(_) => Some(self.witness_script(deriv_ctx)),
+            Descriptor::ShWsh(ref script) => Some(script.encode(deriv_ctx).to_v0_p2wsh()),
+            Descriptor::Sh(ref script) => Some(script.encode(deriv_ctx)),
+            Descriptor::Bare(ref script) => Some(script.encode(deriv_ctx)),
+            Descriptor::ShSortedMulti(ref keys) => Some(keys.encode(deriv_ctx)),
             _ => None,
         }
     }
 
-    fn psbt_witness_script(&self) -> Option<Script> {
+    fn psbt_witness_script(&self, secp: &SecpCtx) -> Option<Script> {
+        let deriv_ctx = descriptor_to_pk_ctx(secp);
+
         match self {
-            Descriptor::Wsh(ref script) => Some(script.encode()),
-            Descriptor::ShWsh(ref script) => Some(script.encode()),
+            Descriptor::Wsh(ref script) => Some(script.encode(deriv_ctx)),
+            Descriptor::ShWsh(ref script) => Some(script.encode(deriv_ctx)),
+            Descriptor::WshSortedMulti(ref keys) | Descriptor::ShWshSortedMulti(ref keys) => {
+                Some(keys.encode(deriv_ctx))
+            }
             _ => None,
         }
     }
@@ -144,22 +283,27 @@ where
 impl DescriptorMeta for Descriptor<DescriptorPublicKey> {
     fn is_witness(&self) -> bool {
         match self {
-            Descriptor::Bare(_) | Descriptor::Pk(_) | Descriptor::Pkh(_) | Descriptor::Sh(_) => {
-                false
-            }
+            Descriptor::Bare(_)
+            | Descriptor::Pk(_)
+            | Descriptor::Pkh(_)
+            | Descriptor::Sh(_)
+            | Descriptor::ShSortedMulti(_) => false,
             Descriptor::Wpkh(_)
             | Descriptor::ShWpkh(_)
             | Descriptor::Wsh(_)
-            | Descriptor::ShWsh(_) => true,
+            | Descriptor::ShWsh(_)
+            | Descriptor::ShWshSortedMulti(_)
+            | Descriptor::WshSortedMulti(_) => true,
         }
     }
 
-    fn get_hd_keypaths(&self, index: u32) -> Result<HDKeyPaths, Error> {
-        let mut answer = BTreeMap::new();
-
-        let translatefpk = |key: &DescriptorPublicKey| -> Result<_, Error> {
+    fn get_hd_keypaths(&self, index: u32, secp: &SecpCtx) -> Result<HDKeyPaths, Error> {
+        let translate_key = |key: &DescriptorPublicKey,
+                             index: u32,
+                             paths: &mut HDKeyPaths|
+         -> Result<DummyKey, Error> {
             match key {
-                DescriptorPublicKey::PubKey(_) => {}
+                DescriptorPublicKey::SinglePub(_) => {}
                 DescriptorPublicKey::XPub(xpub) => {
                     let derive_path = if xpub.is_wildcard {
                         xpub.derivation_path
@@ -174,10 +318,10 @@ impl DescriptorMeta for Descriptor<DescriptorPublicKey> {
                         .xkey
                         .derive_pub(&Secp256k1::verification_only(), &derive_path)?;
 
-                    answer.insert(
+                    paths.insert(
                         derived_pubkey.public_key,
                         (
-                            xpub.root_fingerprint(),
+                            xpub.root_fingerprint(secp),
                             xpub.full_path(&[ChildNumber::from_normal_idx(index)?]),
                         ),
                     );
@@ -186,41 +330,76 @@ impl DescriptorMeta for Descriptor<DescriptorPublicKey> {
 
             Ok(DummyKey::default())
         };
-        let translatefpkh = |_: &hash160::Hash| -> Result<_, Error> { Ok(DummyKey::default()) };
 
-        self.translate_pk(translatefpk, translatefpkh)?;
+        let mut answer_pk = BTreeMap::new();
+        let mut answer_pkh = BTreeMap::new();
 
-        Ok(answer)
+        self.translate_pk(
+            |pk| translate_key(pk, index, &mut answer_pk),
+            |pkh| translate_key(pkh, index, &mut answer_pkh),
+        )?;
+
+        answer_pk.append(&mut answer_pkh);
+
+        Ok(answer_pk)
+    }
+
+    fn get_extended_keys(&self) -> Result<Vec<DescriptorXKey<ExtendedPubKey>>, Error> {
+        let get_key = |key: &DescriptorPublicKey,
+                       keys: &mut Vec<DescriptorXKey<ExtendedPubKey>>|
+         -> Result<DummyKey, Error> {
+            if let DescriptorPublicKey::XPub(xpub) = key {
+                keys.push(xpub.clone())
+            }
+
+            Ok(DummyKey::default())
+        };
+
+        let mut answer_pk = Vec::new();
+        let mut answer_pkh = Vec::new();
+
+        self.translate_pk(
+            |pk| get_key(pk, &mut answer_pk),
+            |pkh| get_key(pkh, &mut answer_pkh),
+        )?;
+
+        answer_pk.append(&mut answer_pkh);
+
+        Ok(answer_pk)
     }
 
     fn is_fixed(&self) -> bool {
-        let mut found_wildcard = false;
-
-        let translatefpk = |key: &DescriptorPublicKey| -> Result<_, Error> {
+        fn check_key(key: &DescriptorPublicKey, flag: &mut bool) -> Result<DummyKey, Error> {
             match key {
-                DescriptorPublicKey::PubKey(_) => {}
+                DescriptorPublicKey::SinglePub(_) => {}
                 DescriptorPublicKey::XPub(xpub) => {
                     if xpub.is_wildcard {
-                        found_wildcard = true;
+                        *flag = true;
                     }
                 }
             }
 
             Ok(DummyKey::default())
-        };
-        let translatefpkh = |_: &hash160::Hash| -> Result<_, Error> { Ok(DummyKey::default()) };
+        }
 
-        self.translate_pk(translatefpk, translatefpkh).unwrap();
+        let mut found_wildcard_pk = false;
+        let mut found_wildcard_pkh = false;
 
-        !found_wildcard
+        self.translate_pk(
+            |pk| check_key(pk, &mut found_wildcard_pk),
+            |pkh| check_key(pkh, &mut found_wildcard_pkh),
+        )
+        .unwrap();
+
+        !found_wildcard_pk && !found_wildcard_pkh
     }
 
-    fn derive_from_hd_keypaths(&self, hd_keypaths: &HDKeyPaths) -> Option<Self> {
-        let index: HashMap<_, _> = hd_keypaths.values().cloned().collect();
-
-        let mut derive_path = None::<DerivationPath>;
-        let translatefpk = |key: &DescriptorPublicKey| -> Result<_, Error> {
-            if derive_path.is_some() {
+    fn derive_from_hd_keypaths(&self, hd_keypaths: &HDKeyPaths, secp: &SecpCtx) -> Option<Self> {
+        let try_key = |key: &DescriptorPublicKey,
+                       index: &HashMap<Fingerprint, DerivationPath>,
+                       found_path: &mut Option<ChildNumber>|
+         -> Result<DummyKey, Error> {
+            if found_path.is_some() {
                 // already found a matching path, we are done
                 return Ok(DummyKey::default());
             }
@@ -228,38 +407,72 @@ impl DescriptorMeta for Descriptor<DescriptorPublicKey> {
             if let DescriptorPublicKey::XPub(xpub) = key {
                 // Check if the key matches one entry in our `index`. If it does, `matches()` will
                 // return the "prefix" that matched, so we remove that prefix from the full path
-                // found in `index` and save it in `derive_path`
-                let root_fingerprint = xpub.root_fingerprint();
-                derive_path = index
+                // found in `index` and save it in `derive_path`. We expect this to be a derivation
+                // path of length 1 if the key `is_wildcard` and an empty path otherwise.
+                let root_fingerprint = xpub.root_fingerprint(secp);
+                let derivation_path: Option<Vec<ChildNumber>> = index
                     .get_key_value(&root_fingerprint)
-                    .and_then(|(fingerprint, path)| xpub.matches(*fingerprint, path))
-                    .map(|prefix_path| prefix_path.into_iter().cloned().collect::<Vec<_>>())
+                    .and_then(|(fingerprint, path)| {
+                        xpub.matches(&(*fingerprint, path.clone()), secp)
+                    })
                     .map(|prefix| {
                         index
-                            .get(&xpub.root_fingerprint())
+                            .get(&xpub.root_fingerprint(secp))
                             .unwrap()
                             .into_iter()
-                            .skip(prefix.len())
+                            .skip(prefix.into_iter().count())
                             .cloned()
                             .collect()
                     });
+
+                match derivation_path {
+                    Some(path) if xpub.is_wildcard && path.len() == 1 => {
+                        *found_path = Some(path[0])
+                    }
+                    Some(path) if !xpub.is_wildcard && path.is_empty() => {
+                        *found_path = Some(ChildNumber::Normal { index: 0 })
+                    }
+                    Some(_) => return Err(Error::InvalidHDKeyPath),
+                    _ => {}
+                }
             }
 
             Ok(DummyKey::default())
         };
-        let translatefpkh = |_: &hash160::Hash| -> Result<_, Error> { Ok(DummyKey::default()) };
 
-        self.translate_pk(translatefpk, translatefpkh).unwrap();
+        let index: HashMap<_, _> = hd_keypaths.values().cloned().collect();
 
-        derive_path.map(|path| self.derive(path.as_ref()))
+        let mut found_path_pk = None;
+        let mut found_path_pkh = None;
+
+        if self
+            .translate_pk(
+                |pk| try_key(pk, &index, &mut found_path_pk),
+                |pkh| try_key(pkh, &index, &mut found_path_pkh),
+            )
+            .is_err()
+        {
+            return None;
+        }
+
+        // if we have found a path for both `found_path_pk` and `found_path_pkh` but they are
+        // different we consider this an error and return None. we only return a path either if
+        // they are equal or if only one of them is Some(_)
+        let merged_path = match (found_path_pk, found_path_pkh) {
+            (Some(a), Some(b)) if a != b => return None,
+            (a, b) => a.or(b),
+        };
+
+        merged_path.map(|path| self.derive(path))
     }
 
     fn derive_from_psbt_input(
         &self,
         psbt_input: &psbt::Input,
         utxo: Option<TxOut>,
+        secp: &SecpCtx,
     ) -> Option<Self> {
-        if let Some(derived) = self.derive_from_hd_keypaths(&psbt_input.hd_keypaths) {
+        if let Some(derived) = self.derive_from_hd_keypaths(&psbt_input.hd_keypaths, secp) {
             return Some(derived);
         } else if !self.is_fixed() {
             // If the descriptor is not fixed we can't brute-force the derivation address, so just
@@ -267,25 +480,44 @@ impl DescriptorMeta for Descriptor<DescriptorPublicKey> {
             return None;
         }
 
+        let deriv_ctx = descriptor_to_pk_ctx(secp);
         match self {
             Descriptor::Pk(_)
             | Descriptor::Pkh(_)
             | Descriptor::Wpkh(_)
             | Descriptor::ShWpkh(_)
                 if utxo.is_some()
-                    && self.script_pubkey() == utxo.as_ref().unwrap().script_pubkey =>
+                    && self.script_pubkey(deriv_ctx) == utxo.as_ref().unwrap().script_pubkey =>
             {
                 Some(self.clone())
             }
-            Descriptor::Bare(ms) | Descriptor::Sh(ms)
+            Descriptor::Bare(ms)
                 if psbt_input.redeem_script.is_some()
-                    && &ms.encode() == psbt_input.redeem_script.as_ref().unwrap() =>
+                    && &ms.encode(deriv_ctx) == psbt_input.redeem_script.as_ref().unwrap() =>
+            {
+                Some(self.clone())
+            }
+            Descriptor::Sh(ms)
+                if psbt_input.redeem_script.is_some()
+                    && &ms.encode(deriv_ctx) == psbt_input.redeem_script.as_ref().unwrap() =>
             {
                 Some(self.clone())
             }
             Descriptor::Wsh(ms) | Descriptor::ShWsh(ms)
                 if psbt_input.witness_script.is_some()
-                    && &ms.encode() == psbt_input.witness_script.as_ref().unwrap() =>
+                    && &ms.encode(deriv_ctx) == psbt_input.witness_script.as_ref().unwrap() =>
+            {
+                Some(self.clone())
+            }
+            Descriptor::ShSortedMulti(keys)
+                if psbt_input.redeem_script.is_some()
+                    && &keys.encode(deriv_ctx) == psbt_input.redeem_script.as_ref().unwrap() =>
+            {
+                Some(self.clone())
+            }
+            Descriptor::WshSortedMulti(keys) | Descriptor::ShWshSortedMulti(keys)
+                if psbt_input.witness_script.is_some()
+                    && &keys.encode(deriv_ctx) == psbt_input.witness_script.as_ref().unwrap() =>
             {
                 Some(self.clone())
             }
@@ -325,7 +557,8 @@ mod test {
 
     use bitcoin::consensus::encode::deserialize;
     use bitcoin::hashes::hex::FromHex;
-    use bitcoin::util::psbt;
+    use bitcoin::secp256k1::Secp256k1;
+    use bitcoin::util::{bip32, psbt};
 
     use super::*;
     use crate::psbt::PSBTUtils;
@@ -349,7 +582,7 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
 
@@ -380,7 +613,7 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
 
@@ -404,7 +637,7 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
 
@@ -434,7 +667,112 @@ mod test {
         .unwrap();
 
         assert!(descriptor
-            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0))
+            .derive_from_psbt_input(&psbt.inputs[0], psbt.get_utxo_for(0), &Secp256k1::new())
             .is_some());
     }
+
+    #[test]
+    fn test_to_wallet_descriptor_fixup_networks() {
+        use crate::keys::{any_network, ToDescriptorKey};
+
+        let xpub = bip32::ExtendedPubKey::from_str("xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL").unwrap();
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+
+        // here `to_descriptor_key` will set the valid networks for the key to only mainnet, since
+        // we are using an "xpub"
+        let key = (xpub, path).to_descriptor_key().unwrap();
+        // override it with any. this happens in some key conversions, like bip39
+        let key = key.override_valid_networks(any_network());
+
+        // make a descriptor out of it
+        let desc = crate::descriptor!(wpkh(key)).unwrap();
+        // this should conver the key that supports "any_network" to the right network (testnet)
+        let (wallet_desc, _) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+
+        assert_eq!(wallet_desc.to_string(), "wpkh(tpubDEnoLuPdBep9bzw5LoGYpsxUQYheRQ9gcgrJhJEcdKFB9cWQRyYmkCyRoTqeD4tJYiVVgt6A3rN6rWn9RYhR9sBsGxji29LYWHuKKbdb1ev/0/*)");
+    }
+
+    // test ToWalletDescriptor trait from &str with and without checksum appended
+    #[test]
+    fn test_descriptor_from_str_with_checksum() {
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#tqz0nc62"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)#67ju93jw"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#67ju93jw"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(matches!(desc.err(), Some(KeyError::InvalidChecksum)));
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#67ju93jw"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(matches!(desc.err(), Some(KeyError::InvalidChecksum)));
+    }
+
+    // test ToWalletDescriptor trait from &str with keys from right and wrong network
+    #[test]
+    fn test_descriptor_from_str_with_keys_network() {
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .to_wallet_descriptor(Network::Regtest);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .to_wallet_descriptor(Network::Regtest);
+        assert!(desc.is_ok());
+
+        let desc = "sh(wpkh(02864bb4ad00cefa806098a69e192bbda937494e69eb452b87bb3f20f6283baedb))"
+            .to_wallet_descriptor(Network::Testnet);
+        assert!(desc.is_ok());
+
+        let desc = "sh(wpkh(02864bb4ad00cefa806098a69e192bbda937494e69eb452b87bb3f20f6283baedb))"
+            .to_wallet_descriptor(Network::Bitcoin);
+        assert!(desc.is_ok());
+
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)"
+            .to_wallet_descriptor(Network::Bitcoin);
+        assert!(matches!(desc.err(), Some(KeyError::InvalidNetwork)));
+
+        let desc = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)"
+            .to_wallet_descriptor(Network::Bitcoin);
+        assert!(matches!(desc.err(), Some(KeyError::InvalidNetwork)));
+    }
+
+    // test ToWalletDescriptor trait from the output of the descriptor!() macro
+    #[test]
+    fn test_descriptor_from_str_from_output_of_macro() {
+        let tpub = bip32::ExtendedPubKey::from_str("tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK").unwrap();
+        let path = bip32::DerivationPath::from_str("m/1/2").unwrap();
+        let key = (tpub, path).to_descriptor_key().unwrap();
+
+        // make a descriptor out of it
+        let desc = crate::descriptor!(wpkh(key)).unwrap();
+
+        let (wallet_desc, _) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let wallet_desc_str = wallet_desc.to_string();
+        assert_eq!(wallet_desc_str, "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/2/*)");
+
+        let (wallet_desc2, _) = wallet_desc_str
+            .to_wallet_descriptor(Network::Testnet)
+            .unwrap();
+        assert_eq!(wallet_desc, wallet_desc2)
+    }
 }

src/descriptor/policy.rs

@@ -27,26 +27,29 @@
 //! This module implements the logic to extract and represent the spending policies of a descriptor
 //! in a more human-readable format.
 //!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
 //! ## Example
 //!
 //! ```
 //! # use std::sync::Arc;
-//! # use magical::descriptor::*;
+//! # use bdk::descriptor::*;
+//! # use bdk::bitcoin::secp256k1::Secp256k1;
+//! let secp = Secp256k1::new();
 //! let desc = "wsh(and_v(v:pk(cV3oCth6zxZ1UVsHLnGothsWNsaoxRhC6aeNi5VbSdFpwUkgkEci),or_d(pk(cVMTy7uebJgvFaSBwcgvwk8qn8xSLc97dKow4MBetjrrahZoimm2),older(12960))))";
 //!
-//! let (extended_desc, key_map) = ExtendedDescriptor::parse_secret(desc)?;
+//! let (extended_desc, key_map) = ExtendedDescriptor::parse_descriptor(desc)?;
 //! println!("{:?}", extended_desc);
 //!
 //! let signers = Arc::new(key_map.into());
-//! let policy = extended_desc.extract_policy(signers)?;
+//! let policy = extended_desc.extract_policy(&signers, &secp)?;
 //! println!("policy: {}", serde_json::to_string(&policy)?);
-//! # Ok::<(), magical::Error>(())
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
-use std::cmp::max;
+use std::cmp::{max, Ordering};
 use std::collections::{BTreeMap, HashSet, VecDeque};
 use std::fmt;
-use std::sync::Arc;
 
 use serde::ser::SerializeMap;
 use serde::{Serialize, Serializer};
@@ -55,14 +58,15 @@ use bitcoin::hashes::*;
 use bitcoin::util::bip32::Fingerprint;
 use bitcoin::PublicKey;
 
-use miniscript::descriptor::DescriptorPublicKey;
-use miniscript::{Descriptor, Miniscript, MiniscriptKey, ScriptContext, Terminal};
+use miniscript::descriptor::{DescriptorPublicKey, SortedMultiVec};
+use miniscript::{Descriptor, Miniscript, MiniscriptKey, ScriptContext, Terminal, ToPublicKey};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
 use crate::descriptor::ExtractPolicy;
 use crate::wallet::signer::{SignerId, SignersContainer};
+use crate::wallet::utils::{self, descriptor_to_pk_ctx, SecpCtx};
 
 use super::checksum::get_checksum;
 use super::error::Error;
@@ -80,14 +84,14 @@ pub struct PKOrF {
 }
 
 impl PKOrF {
-    fn from_key(k: &DescriptorPublicKey) -> Self {
+    fn from_key(k: &DescriptorPublicKey, secp: &SecpCtx) -> Self {
         match k {
-            DescriptorPublicKey::PubKey(pubkey) => PKOrF {
-                pubkey: Some(*pubkey),
+            DescriptorPublicKey::SinglePub(pubkey) => PKOrF {
+                pubkey: Some(pubkey.key),
                 ..Default::default()
             },
             DescriptorPublicKey::XPub(xpub) => PKOrF {
-                fingerprint: Some(xpub.root_fingerprint()),
+                fingerprint: Some(xpub.root_fingerprint(secp)),
                 ..Default::default()
             },
         }
@@ -101,61 +105,81 @@ impl PKOrF {
     }
 }
 
-/// An item that need to be satisfied
+/// An item that needs to be satisfied
 #[derive(Debug, Clone, Serialize)]
 #[serde(tag = "type", rename_all = "UPPERCASE")]
 pub enum SatisfiableItem {
     // Leaves
+    /// Signature for a raw public key
     Signature(PKOrF),
+    /// Signature for an extended key fingerprint
     SignatureKey(PKOrF),
+    /// SHA256 preimage hash
     SHA256Preimage {
+        /// The digest value
         hash: sha256::Hash,
     },
+    /// Double SHA256 preimage hash
     HASH256Preimage {
+        /// The digest value
         hash: sha256d::Hash,
     },
+    /// RIPEMD160 preimage hash
     RIPEMD160Preimage {
+        /// The digest value
         hash: ripemd160::Hash,
     },
+    /// SHA256 then RIPEMD160 preimage hash
     HASH160Preimage {
+        /// The digest value
         hash: hash160::Hash,
     },
+    /// Absolute timeclock timestamp
     AbsoluteTimelock {
+        /// The timestamp value
         value: u32,
     },
+    /// Relative timelock locktime
     RelativeTimelock {
+        /// The locktime value
         value: u32,
     },
+    /// Multi-signature public keys with threshold count
+    Multisig {
+        /// The raw public key or extended key fingerprint
+        keys: Vec<PKOrF>,
+        /// The required threshold count
+        threshold: usize,
+    },
 
     // Complex item
+    /// Threshold items with threshold count
     Thresh {
+        /// The policy items
         items: Vec<Policy>,
-        threshold: usize,
-    },
-    Multisig {
-        keys: Vec<PKOrF>,
+        /// The required threshold count
         threshold: usize,
     },
 }
 
 impl SatisfiableItem {
+    /// Returns whether the [`SatisfiableItem`] is a leaf item
     pub fn is_leaf(&self) -> bool {
-        match self {
-            SatisfiableItem::Thresh {
-                items: _,
-                threshold: _,
-            } => false,
-            _ => true,
-        }
+        !matches!(self,
+        SatisfiableItem::Thresh {
+            items: _,
+            threshold: _,
+        })
     }
 
+    /// Returns a unique id for the [`SatisfiableItem`]
     pub fn id(&self) -> String {
         get_checksum(&serde_json::to_string(self).expect("Failed to serialize a SatisfiableItem"))
             .expect("Failed to compute a SatisfiableItem id")
     }
 }
 
-fn combinations(vec: &Vec<usize>, size: usize) -> Vec<Vec<usize>> {
+fn combinations(vec: &[usize], size: usize) -> Vec<Vec<usize>> {
     assert!(vec.len() >= size);
 
     let mut answer = Vec::new();
@@ -213,7 +237,9 @@ fn mix<T: Clone>(vec: Vec<Vec<T>>) -> Vec<Vec<T>> {
     answer
 }
 
+/// Type for a map of sets of [`Condition`] items keyed by each set's index
 pub type ConditionMap = BTreeMap<usize, HashSet<Condition>>;
+/// Type for a map of folded sets of [`Condition`] items keyed by a vector of the combined set's indexes
 pub type FoldedConditionMap = BTreeMap<Vec<usize>, HashSet<Condition>>;
 
 fn serialize_folded_cond_map<S>(
@@ -243,6 +269,9 @@ pub enum Satisfaction {
         m: usize,
         /// The items that can be satisfied by the descriptor
         items: Vec<usize>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        /// Whether the items are sorted in lexicographic order (used by `sortedmulti`)
+        sorted: Option<bool>,
         #[serde(skip_serializing_if = "BTreeMap::is_empty")]
         /// Extra conditions that also need to be satisfied
         conditions: ConditionMap,
@@ -255,6 +284,9 @@ pub enum Satisfaction {
         m: usize,
         /// The items that can be satisfied by the descriptor
         items: Vec<usize>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        /// Whether the items are sorted in lexicographic order (used by `sortedmulti`)
+        sorted: Option<bool>,
         #[serde(
             serialize_with = "serialize_folded_cond_map",
             skip_serializing_if = "BTreeMap::is_empty"
@@ -273,6 +305,7 @@ pub enum Satisfaction {
 }
 
 impl Satisfaction {
+    /// Returns whether the [`Satisfaction`] is a leaf item
     pub fn is_leaf(&self) -> bool {
         match self {
             Satisfaction::None | Satisfaction::Complete { .. } => true,
@@ -326,6 +359,7 @@ impl Satisfaction {
             m,
             items,
             conditions,
+            sorted,
         } = self
         {
             if items.len() >= *m {
@@ -344,8 +378,8 @@ impl Satisfaction {
                             .map(|i| {
                                 conditions
                                     .get(i)
-                                    .and_then(|set| Some(set.clone().into_iter().collect()))
-                                    .unwrap_or(vec![])
+                                    .map(|set| set.clone().into_iter().collect())
+                                    .unwrap_or_default()
                             })
                             .collect())
                         .into_iter()
@@ -356,7 +390,7 @@ impl Satisfaction {
                     // since the previous step can turn one item of the iterator into multiple ones, we call flatten to expand them out
                     .flatten()
                     // .inspect(|x| println!("flat {:?}", x))
-                    // try to fold all the conditions for this specific combination of indexes/options. if they are not compatibile, try_fold will be Err
+                    // try to fold all the conditions for this specific combination of indexes/options. if they are not compatible, try_fold will be Err
                     .map(|(key, val)| {
                         (
                             key,
@@ -380,6 +414,7 @@ impl Satisfaction {
                     m: *m,
                     items: items.clone(),
                     conditions: map,
+                    sorted: *sorted,
                 };
             }
         }
@@ -418,32 +453,45 @@ pub struct Policy {
 /// An extra condition that must be satisfied but that is out of control of the user
 #[derive(Hash, Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Default, Serialize)]
 pub struct Condition {
+    /// Optional CheckSequenceVerify condition
     #[serde(skip_serializing_if = "Option::is_none")]
     pub csv: Option<u32>,
+    /// Optional timelock condition
     #[serde(skip_serializing_if = "Option::is_none")]
     pub timelock: Option<u32>,
 }
 
 impl Condition {
-    fn merge_timelock(a: u32, b: u32) -> Result<u32, PolicyError> {
-        const BLOCKS_TIMELOCK_THRESHOLD: u32 = 500000000;
-
-        if (a < BLOCKS_TIMELOCK_THRESHOLD) != (b < BLOCKS_TIMELOCK_THRESHOLD) {
+    fn merge_nlocktime(a: u32, b: u32) -> Result<u32, PolicyError> {
+        if (a < utils::BLOCKS_TIMELOCK_THRESHOLD) != (b < utils::BLOCKS_TIMELOCK_THRESHOLD) {
             Err(PolicyError::MixedTimelockUnits)
         } else {
             Ok(max(a, b))
         }
     }
 
-    fn merge(mut self, other: &Condition) -> Result<Self, PolicyError> {
+    fn merge_nsequence(a: u32, b: u32) -> Result<u32, PolicyError> {
+        let mask = utils::SEQUENCE_LOCKTIME_TYPE_FLAG | utils::SEQUENCE_LOCKTIME_MASK;
+
+        let a = a & mask;
+        let b = b & mask;
+
+        if (a < utils::SEQUENCE_LOCKTIME_TYPE_FLAG) != (b < utils::SEQUENCE_LOCKTIME_TYPE_FLAG) {
+            Err(PolicyError::MixedTimelockUnits)
+        } else {
+            Ok(max(a, b))
+        }
+    }
+
+    pub(crate) fn merge(mut self, other: &Condition) -> Result<Self, PolicyError> {
         match (self.csv, other.csv) {
-            (Some(a), Some(b)) => self.csv = Some(Self::merge_timelock(a, b)?),
+            (Some(a), Some(b)) => self.csv = Some(Self::merge_nsequence(a, b)?),
             (None, any) => self.csv = any,
             _ => {}
         }
 
         match (self.timelock, other.timelock) {
-            (Some(a), Some(b)) => self.timelock = Some(Self::merge_timelock(a, b)?),
+            (Some(a), Some(b)) => self.timelock = Some(Self::merge_nlocktime(a, b)?),
             (None, any) => self.timelock = any,
             _ => {}
         }
@@ -451,6 +499,7 @@ impl Condition {
         Ok(self)
     }
 
+    /// Returns `true` if there are no extra conditions to verify
     pub fn is_null(&self) -> bool {
         self.csv.is_none() && self.timelock.is_none()
     }
@@ -459,12 +508,19 @@ impl Condition {
 /// Errors that can happen while extracting and manipulating policies
 #[derive(Debug)]
 pub enum PolicyError {
+    /// Not enough items are selected to satisfy a [`SatisfiableItem::Thresh`]
     NotEnoughItemsSelected(String),
+    /// Too many items are selected to satisfy a [`SatisfiableItem::Thresh`]
     TooManyItemsSelected(String),
+    /// Index out of range for an item to satisfy a [`SatisfiableItem::Thresh`]
     IndexOutOfRange(usize),
+    /// Can not add to an item that is [`Satisfaction::None`] or [`Satisfaction::Complete`]
     AddOnLeaf,
+    /// Can not add to an item that is [`Satisfaction::PartialComplete`]
     AddOnPartialComplete,
+    /// Can not merge CSV or timelock values unless both are less than or both are equal or greater than 500_000_000
     MixedTimelockUnits,
+    /// Incompatible conditions (not currently used)
     IncompatibleConditions,
 }
 
@@ -512,6 +568,7 @@ impl Policy {
             m: threshold,
             items: vec![],
             conditions: Default::default(),
+            sorted: None,
         };
         for (index, item) in items.iter().enumerate() {
             contribution.add(&item.contribution, index)?;
@@ -525,24 +582,27 @@ impl Policy {
     }
 
     fn make_multisig(
-        keys: &Vec<DescriptorPublicKey>,
-        signers: Arc<SignersContainer<DescriptorPublicKey>>,
+        keys: &[DescriptorPublicKey],
+        signers: &SignersContainer,
         threshold: usize,
+        sorted: bool,
+        secp: &SecpCtx,
     ) -> Result<Option<Policy>, PolicyError> {
         if threshold == 0 {
             return Ok(None);
         }
 
-        let parsed_keys = keys.iter().map(|k| PKOrF::from_key(k)).collect();
+        let parsed_keys = keys.iter().map(|k| PKOrF::from_key(k, secp)).collect();
 
         let mut contribution = Satisfaction::Partial {
             n: keys.len(),
             m: threshold,
             items: vec![],
             conditions: Default::default(),
+            sorted: Some(sorted),
         };
         for (index, key) in keys.iter().enumerate() {
-            if let Some(_) = signers.find(signer_id(key)) {
+            if signers.find(signer_id(key, secp)).is_some() {
                 contribution.add(
                     &Satisfaction::Complete {
                         condition: Default::default(),
@@ -582,7 +642,7 @@ impl Policy {
         // if items.len() == threshold, selected can be omitted and we take all of them by default
         let default = match &self.item {
             SatisfiableItem::Thresh { items, threshold } if items.len() == *threshold => {
-                (0..*threshold).into_iter().collect()
+                (0..*threshold).collect()
             }
             _ => vec![],
         };
@@ -608,10 +668,14 @@ impl Policy {
                 // if we have something, make sure we have enough items. note that the user can set
                 // an empty value for this step in case of n-of-n, because `selected` is set to all
                 // the elements above
-                if selected.len() < *threshold {
-                    return Err(PolicyError::NotEnoughItemsSelected(self.id.clone()));
-                } else if selected.len() > *threshold {
-                    return Err(PolicyError::TooManyItemsSelected(self.id.clone()));
+                match selected.len().cmp(threshold) {
+                    Ordering::Less => {
+                        return Err(PolicyError::NotEnoughItemsSelected(self.id.clone()))
+                    }
+                    Ordering::Greater => {
+                        return Err(PolicyError::TooManyItemsSelected(self.id.clone()))
+                    }
+                    Ordering::Equal => (),
                 }
 
                 // check the selected items, see if there are conflicting requirements
@@ -646,20 +710,17 @@ impl From<SatisfiableItem> for Policy {
     }
 }
 
-fn signer_id(key: &DescriptorPublicKey) -> SignerId<DescriptorPublicKey> {
+fn signer_id(key: &DescriptorPublicKey, secp: &SecpCtx) -> SignerId {
     match key {
-        DescriptorPublicKey::PubKey(pubkey) => pubkey.to_pubkeyhash().into(),
-        DescriptorPublicKey::XPub(xpub) => xpub.root_fingerprint().into(),
+        DescriptorPublicKey::SinglePub(pubkey) => pubkey.key.to_pubkeyhash().into(),
+        DescriptorPublicKey::XPub(xpub) => xpub.root_fingerprint(secp).into(),
     }
 }
 
-fn signature(
-    key: &DescriptorPublicKey,
-    signers: Arc<SignersContainer<DescriptorPublicKey>>,
-) -> Policy {
-    let mut policy: Policy = SatisfiableItem::Signature(PKOrF::from_key(key)).into();
+fn signature(key: &DescriptorPublicKey, signers: &SignersContainer, secp: &SecpCtx) -> Policy {
+    let mut policy: Policy = SatisfiableItem::Signature(PKOrF::from_key(key, secp)).into();
 
-    policy.contribution = if signers.find(signer_id(key)).is_some() {
+    policy.contribution = if signers.find(signer_id(key, secp)).is_some() {
         Satisfaction::Complete {
             condition: Default::default(),
         }
@@ -671,12 +732,15 @@ fn signature(
 }
 
 fn signature_key(
-    key_hash: &<DescriptorPublicKey as MiniscriptKey>::Hash,
-    signers: Arc<SignersContainer<DescriptorPublicKey>>,
+    key: &<DescriptorPublicKey as MiniscriptKey>::Hash,
+    signers: &SignersContainer,
+    secp: &SecpCtx,
 ) -> Policy {
-    let mut policy: Policy = SatisfiableItem::Signature(PKOrF::from_key_hash(*key_hash)).into();
+    let deriv_ctx = descriptor_to_pk_ctx(secp);
+    let key_hash = key.to_public_key(deriv_ctx).to_pubkeyhash();
+    let mut policy: Policy = SatisfiableItem::Signature(PKOrF::from_key_hash(key_hash)).into();
 
-    if let Some(_) = signers.find(SignerId::PkHash(*key_hash)) {
+    if signers.find(SignerId::PkHash(key_hash)).is_some() {
         policy.contribution = Satisfaction::Complete {
             condition: Default::default(),
         }
@@ -688,13 +752,14 @@ fn signature_key(
 impl<Ctx: ScriptContext> ExtractPolicy for Miniscript<DescriptorPublicKey, Ctx> {
     fn extract_policy(
         &self,
-        signers: Arc<SignersContainer<DescriptorPublicKey>>,
+        signers: &SignersContainer,
+        secp: &SecpCtx,
     ) -> Result<Option<Policy>, Error> {
         Ok(match &self.node {
             // Leaves
             Terminal::True | Terminal::False => None,
-            Terminal::PkK(pubkey) => Some(signature(pubkey, Arc::clone(&signers))),
-            Terminal::PkH(pubkey_hash) => Some(signature_key(pubkey_hash, Arc::clone(&signers))),
+            Terminal::PkK(pubkey) => Some(signature(pubkey, signers, secp)),
+            Terminal::PkH(pubkey_hash) => Some(signature_key(pubkey_hash, signers, secp)),
             Terminal::After(value) => {
                 let mut policy: Policy = SatisfiableItem::AbsoluteTimelock { value: *value }.into();
                 policy.contribution = Satisfaction::Complete {
@@ -727,7 +792,7 @@ impl<Ctx: ScriptContext> ExtractPolicy for Miniscript<DescriptorPublicKey, Ctx>
             Terminal::Hash160(hash) => {
                 Some(SatisfiableItem::HASH160Preimage { hash: *hash }.into())
             }
-            Terminal::Multi(k, pks) => Policy::make_multisig(pks, Arc::clone(&signers), *k)?,
+            Terminal::Multi(k, pks) => Policy::make_multisig(pks, signers, *k, false, secp)?,
             // Identities
             Terminal::Alt(inner)
             | Terminal::Swap(inner)
@@ -735,31 +800,31 @@ impl<Ctx: ScriptContext> ExtractPolicy for Miniscript<DescriptorPublicKey, Ctx>
             | Terminal::DupIf(inner)
             | Terminal::Verify(inner)
             | Terminal::NonZero(inner)
-            | Terminal::ZeroNotEqual(inner) => inner.extract_policy(Arc::clone(&signers))?,
+            | Terminal::ZeroNotEqual(inner) => inner.extract_policy(signers, secp)?,
             // Complex policies
             Terminal::AndV(a, b) | Terminal::AndB(a, b) => Policy::make_and(
-                a.extract_policy(Arc::clone(&signers))?,
-                b.extract_policy(Arc::clone(&signers))?,
+                a.extract_policy(signers, secp)?,
+                b.extract_policy(signers, secp)?,
             )?,
             Terminal::AndOr(x, y, z) => Policy::make_or(
                 Policy::make_and(
-                    x.extract_policy(Arc::clone(&signers))?,
-                    y.extract_policy(Arc::clone(&signers))?,
+                    x.extract_policy(signers, secp)?,
+                    y.extract_policy(signers, secp)?,
                 )?,
-                z.extract_policy(Arc::clone(&signers))?,
+                z.extract_policy(signers, secp)?,
             )?,
             Terminal::OrB(a, b)
             | Terminal::OrD(a, b)
             | Terminal::OrC(a, b)
             | Terminal::OrI(a, b) => Policy::make_or(
-                a.extract_policy(Arc::clone(&signers))?,
-                b.extract_policy(Arc::clone(&signers))?,
+                a.extract_policy(signers, secp)?,
+                b.extract_policy(signers, secp)?,
             )?,
             Terminal::Thresh(k, nodes) => {
                 let mut threshold = *k;
                 let mapped: Vec<_> = nodes
                     .iter()
-                    .map(|n| n.extract_policy(Arc::clone(&signers)))
+                    .map(|n| n.extract_policy(signers, secp))
                     .collect::<Result<Vec<_>, _>>()?
                     .into_iter()
                     .filter_map(|x| x)
@@ -781,15 +846,386 @@ impl<Ctx: ScriptContext> ExtractPolicy for Miniscript<DescriptorPublicKey, Ctx>
 impl ExtractPolicy for Descriptor<DescriptorPublicKey> {
     fn extract_policy(
         &self,
-        signers: Arc<SignersContainer<DescriptorPublicKey>>,
+        signers: &SignersContainer,
+        secp: &SecpCtx,
     ) -> Result<Option<Policy>, Error> {
+        fn make_sortedmulti<Ctx: ScriptContext>(
+            keys: &SortedMultiVec<DescriptorPublicKey, Ctx>,
+            signers: &SignersContainer,
+            secp: &SecpCtx,
+        ) -> Result<Option<Policy>, Error> {
+            Ok(Policy::make_multisig(
+                keys.pks.as_ref(),
+                signers,
+                keys.k,
+                true,
+                secp,
+            )?)
+        }
+
         match self {
             Descriptor::Pk(pubkey)
             | Descriptor::Pkh(pubkey)
             | Descriptor::Wpkh(pubkey)
-            | Descriptor::ShWpkh(pubkey) => Ok(Some(signature(pubkey, signers))),
-            Descriptor::Bare(inner) | Descriptor::Sh(inner) => Ok(inner.extract_policy(signers)?),
-            Descriptor::Wsh(inner) | Descriptor::ShWsh(inner) => Ok(inner.extract_policy(signers)?),
+            | Descriptor::ShWpkh(pubkey) => Ok(Some(signature(pubkey, signers, secp))),
+            Descriptor::Bare(inner) => Ok(inner.extract_policy(signers, secp)?),
+            Descriptor::Sh(inner) => Ok(inner.extract_policy(signers, secp)?),
+            Descriptor::Wsh(inner) | Descriptor::ShWsh(inner) => {
+                Ok(inner.extract_policy(signers, secp)?)
+            }
+
+            // `sortedmulti()` is handled separately
+            Descriptor::ShSortedMulti(keys) => make_sortedmulti(&keys, signers, secp),
+            Descriptor::ShWshSortedMulti(keys) | Descriptor::WshSortedMulti(keys) => {
+                make_sortedmulti(&keys, signers, secp)
+            }
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+
+    use crate::descriptor;
+    use crate::descriptor::{ExtractPolicy, ToWalletDescriptor};
+
+    use super::*;
+    use crate::descriptor::policy::SatisfiableItem::{Multisig, Signature, Thresh};
+    use crate::keys::{DescriptorKey, ToDescriptorKey};
+    use crate::wallet::signer::SignersContainer;
+    use bitcoin::secp256k1::{All, Secp256k1};
+    use bitcoin::util::bip32;
+    use bitcoin::util::bip32::ChildNumber;
+    use bitcoin::Network;
+    use std::str::FromStr;
+    use std::sync::Arc;
+
+    const TPRV0_STR:&str = "tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf";
+    const TPRV1_STR:&str = "tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N";
+
+    const PATH: &str = "m/44'/1'/0'/0";
+
+    fn setup_keys<Ctx: ScriptContext>(
+        tprv: &str,
+    ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
+        let secp: Secp256k1<All> = Secp256k1::new();
+        let path = bip32::DerivationPath::from_str(PATH).unwrap();
+        let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
+        let tpub = bip32::ExtendedPubKey::from_private(&secp, &tprv);
+        let fingerprint = tprv.fingerprint(&secp);
+        let prvkey = (tprv, path.clone()).to_descriptor_key().unwrap();
+        let pubkey = (tpub, path).to_descriptor_key().unwrap();
+
+        (prvkey, pubkey, fingerprint)
+    }
+
+    // test ExtractPolicy trait for simple descriptors; wpkh(), sh(multi())
+
+    #[test]
+    fn test_extract_policy_for_wpkh() {
+        let (prvkey, pubkey, fingerprint) = setup_keys(TPRV0_STR);
+        let desc = descriptor!(wpkh(pubkey)).unwrap();
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = wallet_desc
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Signature(pk_or_f) if &pk_or_f.fingerprint.unwrap() == &fingerprint)
+        );
+        assert!(matches!(&policy.contribution, Satisfaction::None));
+
+        let desc = descriptor!(wpkh(prvkey)).unwrap();
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = wallet_desc
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Signature(pk_or_f) if &pk_or_f.fingerprint.unwrap() == &fingerprint)
+        );
+        assert!(
+            matches!(&policy.contribution, Satisfaction::Complete {condition} if condition.csv == None && condition.timelock == None)
+        );
+    }
+
+    // 2 pub keys descriptor, required 2 prv keys
+    // #[test]
+    // fn test_extract_policy_for_sh_multi_partial_0of2() {
+    //     let (_prvkey0, pubkey0, fingerprint0) = setup_keys(TPRV0_STR);
+    //     let (_prvkey1, pubkey1, fingerprint1) = setup_keys(TPRV1_STR);
+    //     let desc = descriptor!(sh(multi 2, pubkey0, pubkey1)).unwrap();
+    //     let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+    //     let signers_container = Arc::new(SignersContainer::from(keymap));
+    //     let policy = wallet_desc
+    //         .extract_policy(signers_container)
+    //         .unwrap()
+    //         .unwrap();
+    //
+    //     assert!(
+    //         matches!(&policy.item, Multisig { keys, threshold } if threshold == &2
+    //         && &keys[0].fingerprint.unwrap() == &fingerprint0
+    //         && &keys[1].fingerprint.unwrap() == &fingerprint1)
+    //     );
+    //
+    //     // TODO should this be "Satisfaction::None" since we have no prv keys?
+    //     // TODO should items and conditions not be empty?
+    //     assert!(
+    //         matches!(&policy.contribution, Satisfaction::Partial { n, m, items, conditions} if n == &2
+    //         && m == &2
+    //         && items.is_empty()
+    //         && conditions.is_empty()
+    //         )
+    //     );
+    // }
+
+    // 1 prv and 1 pub key descriptor, required 2 prv keys
+    // #[test]
+    // fn test_extract_policy_for_sh_multi_partial_1of2() {
+    //     let (prvkey0, _pubkey0, fingerprint0) = setup_keys(TPRV0_STR);
+    //     let (_prvkey1, pubkey1, fingerprint1) = setup_keys(TPRV1_STR);
+    //     let desc = descriptor!(sh(multi 2, prvkey0, pubkey1)).unwrap();
+    //     let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+    //     let signers_container = Arc::new(SignersContainer::from(keymap));
+    //     let policy = wallet_desc
+    //         .extract_policy(signers_container)
+    //         .unwrap()
+    //         .unwrap();
+    //
+    //     assert!(
+    //         matches!(&policy.item, Multisig { keys, threshold } if threshold == &2
+    //         && &keys[0].fingerprint.unwrap() == &fingerprint0
+    //         && &keys[1].fingerprint.unwrap() == &fingerprint1)
+    //     );
+    //
+    //     // TODO should this be "Satisfaction::Partial" since we have only one of two prv keys?
+    //     assert!(
+    //         matches!(&policy.contribution, Satisfaction::PartialComplete { n, m, items, conditions} if n == &2
+    //          && m == &2
+    //          && items.len() == 2
+    //          && conditions.contains_key(&vec![0,1])
+    //         )
+    //     );
+    // }
+
+    // 1 prv and 1 pub key descriptor, required 1 prv keys
+    #[test]
+    #[ignore] // see https://github.com/bitcoindevkit/bdk/issues/225
+    fn test_extract_policy_for_sh_multi_complete_1of2() {
+        let (_prvkey0, pubkey0, fingerprint0) = setup_keys(TPRV0_STR);
+        let (prvkey1, _pubkey1, fingerprint1) = setup_keys(TPRV1_STR);
+        let desc = descriptor!(sh(multi(1, pubkey0, prvkey1))).unwrap();
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = wallet_desc
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Multisig { keys, threshold } if threshold == &1
+            && &keys[0].fingerprint.unwrap() == &fingerprint0
+            && &keys[1].fingerprint.unwrap() == &fingerprint1)
+        );
+        assert!(
+            matches!(&policy.contribution, Satisfaction::PartialComplete { n, m, items, conditions, .. } if n == &2
+             && m == &1
+             && items.len() == 2
+             && conditions.contains_key(&vec![0])
+             && conditions.contains_key(&vec![1])
+            )
+        );
+    }
+
+    // 2 prv keys descriptor, required 2 prv keys
+    #[test]
+    fn test_extract_policy_for_sh_multi_complete_2of2() {
+        let (prvkey0, _pubkey0, fingerprint0) = setup_keys(TPRV0_STR);
+        let (prvkey1, _pubkey1, fingerprint1) = setup_keys(TPRV1_STR);
+        let desc = descriptor!(sh(multi(2, prvkey0, prvkey1))).unwrap();
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = wallet_desc
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Multisig { keys, threshold } if threshold == &2
+            && &keys[0].fingerprint.unwrap() == &fingerprint0
+            && &keys[1].fingerprint.unwrap() == &fingerprint1)
+        );
+
+        assert!(
+            matches!(&policy.contribution, Satisfaction::PartialComplete { n, m, items, conditions, .. } if n == &2
+             && m == &2
+             && items.len() == 2
+             && conditions.contains_key(&vec![0,1])
+            )
+        );
+    }
+
+    // test ExtractPolicy trait with extended and single keys
+
+    #[test]
+    fn test_extract_policy_for_single_wpkh() {
+        let (prvkey, pubkey, fingerprint) = setup_keys(TPRV0_STR);
+        let desc = descriptor!(wpkh(pubkey)).unwrap();
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let single_key = wallet_desc.derive(ChildNumber::from_normal_idx(0).unwrap());
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = single_key
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Signature(pk_or_f) if &pk_or_f.fingerprint.unwrap() == &fingerprint)
+        );
+        assert!(matches!(&policy.contribution, Satisfaction::None));
+
+        let desc = descriptor!(wpkh(prvkey)).unwrap();
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let single_key = wallet_desc.derive(ChildNumber::from_normal_idx(0).unwrap());
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = single_key
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Signature(pk_or_f) if &pk_or_f.fingerprint.unwrap() == &fingerprint)
+        );
+        assert!(
+            matches!(&policy.contribution, Satisfaction::Complete {condition} if condition.csv == None && condition.timelock == None)
+        );
+    }
+
+    // single key, 1 prv and 1 pub key descriptor, required 1 prv keys
+    #[test]
+    #[ignore] // see https://github.com/bitcoindevkit/bdk/issues/225
+    fn test_extract_policy_for_single_wsh_multi_complete_1of2() {
+        let (_prvkey0, pubkey0, fingerprint0) = setup_keys(TPRV0_STR);
+        let (prvkey1, _pubkey1, fingerprint1) = setup_keys(TPRV1_STR);
+        let desc = descriptor!(sh(multi(1, pubkey0, prvkey1))).unwrap();
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let single_key = wallet_desc.derive(ChildNumber::from_normal_idx(0).unwrap());
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = single_key
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Multisig { keys, threshold } if threshold == &1
+            && &keys[0].fingerprint.unwrap() == &fingerprint0
+            && &keys[1].fingerprint.unwrap() == &fingerprint1)
+        );
+        assert!(
+            matches!(&policy.contribution, Satisfaction::PartialComplete { n, m, items, conditions, .. } if n == &2
+             && m == &1
+             && items.len() == 2
+             && conditions.contains_key(&vec![0])
+             && conditions.contains_key(&vec![1])
+            )
+        );
+    }
+
+    // test ExtractPolicy trait with descriptors containing timelocks in a thresh()
+
+    #[test]
+    #[ignore] // see https://github.com/bitcoindevkit/bdk/issues/225
+    fn test_extract_policy_for_wsh_multi_timelock() {
+        let (prvkey0, _pubkey0, _fingerprint0) = setup_keys(TPRV0_STR);
+        let (_prvkey1, pubkey1, _fingerprint1) = setup_keys(TPRV1_STR);
+        let sequence = 50;
+        let desc = descriptor!(wsh(thresh(
+            2,
+            pk(prvkey0),
+            s: pk(pubkey1),
+            s: d: v: older(sequence)
+        )))
+        .unwrap();
+
+        let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+        let signers_container = Arc::new(SignersContainer::from(keymap));
+        let policy = wallet_desc
+            .extract_policy(&signers_container, &Secp256k1::new())
+            .unwrap()
+            .unwrap();
+
+        assert!(
+            matches!(&policy.item, Thresh { items, threshold } if items.len() == 3 && threshold == &2)
+        );
+
+        assert!(
+            matches!(&policy.contribution, Satisfaction::PartialComplete { n, m, items, conditions, .. } if n == &3
+             && m == &2
+             && items.len() == 3
+             && conditions.get(&vec![0,1]).unwrap().iter().next().unwrap().csv.is_none()
+             && conditions.get(&vec![0,2]).unwrap().iter().next().unwrap().csv == Some(sequence)
+             && conditions.get(&vec![1,2]).unwrap().iter().next().unwrap().csv == Some(sequence)
+            )
+        );
+    }
+
+    // - mixed timelocks should fail
+
+    // #[test]
+    // fn test_extract_policy_for_wsh_mixed_timelocks() {
+    //     let (prvkey0, _pubkey0, _fingerprint0) = setup_keys(TPRV0_STR);
+    //     let locktime_threshold = 500000000; // if less than this means block number, else block time in seconds
+    //     let locktime_blocks = 100;
+    //     let locktime_seconds = locktime_blocks + locktime_threshold;
+    //     let desc = descriptor!(sh (and_v (+v pk prvkey0), (and_v (+v after locktime_seconds), (after locktime_blocks)))).unwrap();
+    //     let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+    //     let signers_container = Arc::new(SignersContainer::from(keymap));
+    //     let policy = wallet_desc
+    //         .extract_policy(signers_container)
+    //         .unwrap()
+    //         .unwrap();
+    //
+    //     println!("desc policy = {:?}", policy); // TODO remove
+    //
+    //     // TODO how should this fail with mixed timelocks?
+    // }
+
+    // - multiple timelocks of the same type should be correctly merged together
+
+    // #[test]
+    // fn test_extract_policy_for_multiple_same_timelocks() {
+    //     let (prvkey0, _pubkey0, _fingerprint0) = setup_keys(TPRV0_STR);
+    //     let locktime_blocks0 = 100;
+    //     let locktime_blocks1 = 200;
+    //     let desc = descriptor!(sh (and_v (+v pk prvkey0), (and_v (+v after locktime_blocks0), (after locktime_blocks1)))).unwrap();
+    //     let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+    //     let signers_container = Arc::new(SignersContainer::from(keymap));
+    //     let policy = wallet_desc
+    //         .extract_policy(signers_container)
+    //         .unwrap()
+    //         .unwrap();
+    //
+    //     println!("desc policy = {:?}", policy); // TODO remove
+    //
+    //     // TODO how should this merge timelocks?
+    //
+    //     let (prvkey1, _pubkey1, _fingerprint1) = setup_keys(TPRV0_STR);
+    //     let locktime_seconds0 = 500000100;
+    //     let locktime_seconds1 = 500000200;
+    //     let desc = descriptor!(sh (and_v (+v pk prvkey1), (and_v (+v after locktime_seconds0), (after locktime_seconds1)))).unwrap();
+    //     let (wallet_desc, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+    //     let signers_container = Arc::new(SignersContainer::from(keymap));
+    //     let policy = wallet_desc
+    //         .extract_policy(signers_container)
+    //         .unwrap()
+    //         .unwrap();
+    //
+    //     println!("desc policy = {:?}", policy); // TODO remove
+    //
+    //     // TODO how should this merge timelocks?
+    // }
+}

src/descriptor/template.rs

@@ -0,0 +1,727 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Descriptor templates
+//!
+//! This module contains the definition of various common script templates that are ready to be
+//! used. See the documentation of each template for an example.
+
+use bitcoin::util::bip32;
+use bitcoin::Network;
+
+use miniscript::{Legacy, Segwitv0};
+
+use super::{ExtendedDescriptor, KeyMap, ToWalletDescriptor};
+use crate::keys::{DerivableKey, KeyError, ToDescriptorKey, ValidNetworks};
+use crate::{descriptor, KeychainKind};
+
+/// Type alias for the return type of [`DescriptorTemplate`], [`descriptor!`](crate::descriptor!) and others
+pub type DescriptorTemplateOut = (ExtendedDescriptor, KeyMap, ValidNetworks);
+
+/// Trait for descriptor templates that can be built into a full descriptor
+///
+/// Since [`ToWalletDescriptor`] is implemented for any [`DescriptorTemplate`], they can also be
+/// passed directly to the [`Wallet`](crate::Wallet) constructor.
+///
+/// ## Example
+///
+/// ```
+/// use bdk::keys::{KeyError, ToDescriptorKey};
+/// use bdk::miniscript::Legacy;
+/// use bdk::template::{DescriptorTemplate, DescriptorTemplateOut};
+///
+/// struct MyP2PKH<K: ToDescriptorKey<Legacy>>(K);
+///
+/// impl<K: ToDescriptorKey<Legacy>> DescriptorTemplate for MyP2PKH<K> {
+///     fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+///         Ok(bdk::descriptor!(pkh(self.0))?)
+///     }
+/// }
+/// ```
+pub trait DescriptorTemplate {
+    /// Build the complete descriptor
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError>;
+}
+
+/// Turns a [`DescriptorTemplate`] into a valid wallet descriptor by calling its
+/// [`build`](DescriptorTemplate::build) method
+impl<T: DescriptorTemplate> ToWalletDescriptor for T {
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError> {
+        Ok(self.build()?.to_wallet_descriptor(network)?)
+    }
+}
+
+/// P2PKH template. Expands to a descriptor `pkh(key)`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::P2PKH;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     P2PKH(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_new_address()?.to_string(),
+///     "mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct P2PKH<K: ToDescriptorKey<Legacy>>(pub K);
+
+impl<K: ToDescriptorKey<Legacy>> DescriptorTemplate for P2PKH<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(descriptor!(pkh(self.0))?)
+    }
+}
+
+/// P2WPKH-P2SH template. Expands to a descriptor `sh(wpkh(key))`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::P2WPKH_P2SH;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     P2WPKH_P2SH(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_new_address()?.to_string(),
+///     "2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+#[allow(non_camel_case_types)]
+pub struct P2WPKH_P2SH<K: ToDescriptorKey<Segwitv0>>(pub K);
+
+impl<K: ToDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH_P2SH<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(descriptor!(sh(wpkh(self.0)))?)
+    }
+}
+
+/// P2WPKH template. Expands to a descriptor `wpkh(key)`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::P2WPKH;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     P2WPKH(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_new_address()?.to_string(),
+///     "tb1q4525hmgw265tl3drrl8jjta7ayffu6jf68ltjd"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct P2WPKH<K: ToDescriptorKey<Segwitv0>>(pub K);
+
+impl<K: ToDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(descriptor!(wpkh(self.0))?)
+    }
+}
+
+/// BIP44 template. Expands to `pkh(key/44'/0'/0'/{0,1}/*)`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`BIP44Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP44;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP44(key.clone(), KeychainKind::External),
+///     Some(BIP44(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP44<K: DerivableKey<Legacy>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2PKH(legacy::make_bipxx_private(44, self.0, self.1)?).build()?)
+    }
+}
+
+/// BIP44 public template. Expands to `pkh(key/{0,1}/*)`
+///
+/// This assumes that the key used has already been derived with `m/44'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`BIP44`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP44Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP44Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(BIP44Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP44Public<K: DerivableKey<Legacy>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2PKH(legacy::make_bipxx_public(44, self.0, self.1, self.2)?).build()?)
+    }
+}
+
+/// BIP49 template. Expands to `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`BIP49Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP49;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP49(key.clone(), KeychainKind::External),
+///     Some(BIP49(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP49<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH_P2SH(segwit_v0::make_bipxx_private(49, self.0, self.1)?).build()?)
+    }
+}
+
+/// BIP49 public template. Expands to `sh(wpkh(key/{0,1}/*))`
+///
+/// This assumes that the key used has already been derived with `m/49'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`BIP49`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP49Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP49Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(BIP49Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP49Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH_P2SH(segwit_v0::make_bipxx_public(49, self.0, self.1, self.2)?).build()?)
+    }
+}
+
+/// BIP84 template. Expands to `wpkh(key/84'/0'/0'/{0,1}/*)`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`BIP84Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP84;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP84(key.clone(), KeychainKind::External),
+///     Some(BIP84(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP84<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH(segwit_v0::make_bipxx_private(84, self.0, self.1)?).build()?)
+    }
+}
+
+/// BIP84 public template. Expands to `wpkh(key/{0,1}/*)`
+///
+/// This assumes that the key used has already been derived with `m/84'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`BIP84`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP84Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP84Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(BIP84Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP84Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH(segwit_v0::make_bipxx_public(84, self.0, self.1, self.2)?).build()?)
+    }
+}
+
+macro_rules! expand_make_bipxx {
+    ( $mod_name:ident, $ctx:ty ) => {
+        mod $mod_name {
+            use super::*;
+
+            pub(super) fn make_bipxx_private<K: DerivableKey<$ctx>>(
+                bip: u32,
+                key: K,
+                keychain: KeychainKind,
+            ) -> Result<impl ToDescriptorKey<$ctx>, KeyError> {
+                let mut derivation_path = Vec::with_capacity(4);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(bip)?);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+
+                match keychain {
+                    KeychainKind::External => {
+                        derivation_path.push(bip32::ChildNumber::from_normal_idx(0)?)
+                    }
+                    KeychainKind::Internal => {
+                        derivation_path.push(bip32::ChildNumber::from_normal_idx(1)?)
+                    }
+                };
+
+                let derivation_path: bip32::DerivationPath = derivation_path.into();
+
+                Ok((key, derivation_path))
+            }
+            pub(super) fn make_bipxx_public<K: DerivableKey<$ctx>>(
+                bip: u32,
+                key: K,
+                parent_fingerprint: bip32::Fingerprint,
+                keychain: KeychainKind,
+            ) -> Result<impl ToDescriptorKey<$ctx>, KeyError> {
+                let derivation_path: bip32::DerivationPath = match keychain {
+                    KeychainKind::External => vec![bip32::ChildNumber::from_normal_idx(0)?].into(),
+                    KeychainKind::Internal => vec![bip32::ChildNumber::from_normal_idx(1)?].into(),
+                };
+
+                let mut source_path = Vec::with_capacity(3);
+                source_path.push(bip32::ChildNumber::from_hardened_idx(bip)?);
+                source_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+                source_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+                let source_path: bip32::DerivationPath = source_path.into();
+
+                Ok((key, (parent_fingerprint, source_path), derivation_path))
+            }
+        }
+    };
+}
+
+expand_make_bipxx!(legacy, Legacy);
+expand_make_bipxx!(segwit_v0, Segwitv0);
+
+#[cfg(test)]
+mod test {
+    // test existing descriptor templates, make sure they are expanded to the right descriptors
+
+    use super::*;
+    use crate::descriptor::DescriptorMeta;
+    use crate::keys::{KeyError, ValidNetworks};
+    use bitcoin::hashes::core::str::FromStr;
+    use bitcoin::network::constants::Network::Regtest;
+    use bitcoin::secp256k1::Secp256k1;
+    use bitcoin::util::bip32::ChildNumber;
+    use miniscript::descriptor::{DescriptorPublicKey, DescriptorPublicKeyCtx, KeyMap};
+    use miniscript::Descriptor;
+
+    // verify template descriptor generates expected address(es)
+    fn check(
+        desc: Result<(Descriptor<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError>,
+        is_witness: bool,
+        is_fixed: bool,
+        expected: &[&str],
+    ) {
+        let secp = Secp256k1::new();
+        let deriv_ctx =
+            DescriptorPublicKeyCtx::new(&secp, ChildNumber::from_normal_idx(0).unwrap());
+
+        let (desc, _key_map, _networks) = desc.unwrap();
+        assert_eq!(desc.is_witness(), is_witness);
+        assert_eq!(desc.is_fixed(), is_fixed);
+        for i in 0..expected.len() {
+            let index = i as u32;
+            let child_desc = if desc.is_fixed() {
+                desc.clone()
+            } else {
+                desc.derive(ChildNumber::from_normal_idx(index).unwrap())
+            };
+            let address = child_desc.address(Regtest, deriv_ctx).unwrap();
+            assert_eq!(address.to_string(), *expected.get(i).unwrap());
+        }
+    }
+
+    // P2PKH
+    #[test]
+    fn test_p2ph_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2PKH(prvkey).build(),
+            false,
+            true,
+            &["mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2PKH(pubkey).build(),
+            false,
+            true,
+            &["muZpTpBYhxmRFuCjLc7C6BBDF32C8XVJUi"],
+        );
+    }
+
+    // P2WPKH-P2SH `sh(wpkh(key))`
+    #[test]
+    fn test_p2wphp2sh_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2WPKH_P2SH(prvkey).build(),
+            true,
+            true,
+            &["2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2WPKH_P2SH(pubkey).build(),
+            true,
+            true,
+            &["2N5LiC3CqzxDamRTPG1kiNv1FpNJQ7x28sb"],
+        );
+    }
+
+    // P2WPKH `wpkh(key)`
+    #[test]
+    fn test_p2wph_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2WPKH(prvkey).build(),
+            true,
+            true,
+            &["bcrt1q4525hmgw265tl3drrl8jjta7ayffu6jfcwxx9y"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2WPKH(pubkey).build(),
+            true,
+            true,
+            &["bcrt1qngw83fg8dz0k749cg7k3emc7v98wy0c7azaa6h"],
+        );
+    }
+
+    // BIP44 `pkh(key/44'/0'/0'/{0,1}/*)`
+    #[test]
+    fn test_bip44_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            BIP44(prvkey, KeychainKind::External).build(),
+            false,
+            false,
+            &[
+                "n453VtnjDHPyDt2fDstKSu7A3YCJoHZ5g5",
+                "mvfrrumXgTtwFPWDNUecBBgzuMXhYM7KRP",
+                "mzYvhRAuQqbdSKMVVzXNYyqihgNdRadAUQ",
+            ],
+        );
+        check(
+            BIP44(prvkey, KeychainKind::Internal).build(),
+            false,
+            false,
+            &[
+                "muHF98X9KxEzdKrnFAX85KeHv96eXopaip",
+                "n4hpyLJE5ub6B5Bymv4eqFxS5KjrewSmYR",
+                "mgvkdv1ffmsXd2B1sRKQ5dByK3SzpG42rA",
+            ],
+        );
+    }
+
+    // BIP44 public `pkh(key/{0,1}/*)`
+    #[test]
+    fn test_bip44_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            BIP44Public(pubkey, fingerprint, KeychainKind::External).build(),
+            false,
+            false,
+            &[
+                "miNG7dJTzJqNbFS19svRdTCisC65dsubtR",
+                "n2UqaDbCjWSFJvpC84m3FjUk5UaeibCzYg",
+                "muCPpS6Ue7nkzeJMWDViw7Lkwr92Yc4K8g",
+            ],
+        );
+        check(
+            BIP44Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            false,
+            false,
+            &[
+                "moDr3vJ8wpt5nNxSK55MPq797nXJb2Ru9H",
+                "ms7A1Yt4uTezT2XkefW12AvLoko8WfNJMG",
+                "mhYiyat2rtEnV77cFfQsW32y1m2ceCGHPo",
+            ],
+        );
+    }
+
+    // BIP49 `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
+    #[test]
+    fn test_bip49_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            BIP49(prvkey, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "2N9bCAJXGm168MjVwpkBdNt6ucka3PKVoUV",
+                "2NDckYkqrYyDMtttEav5hB3Bfw9EGAW5HtS",
+                "2NAFTVtksF9T4a97M7nyCjwUBD24QevZ5Z4",
+            ],
+        );
+        check(
+            BIP49(prvkey, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "2NB3pA8PnzJLGV8YEKNDFpbViZv3Bm1K6CG",
+                "2NBiX2Wzxngb5rPiWpUiJQ2uLVB4HBjFD4p",
+                "2NA8ek4CdQ6aMkveYF6AYuEYNrftB47QGTn",
+            ],
+        );
+    }
+
+    // BIP49 public `sh(wpkh(key/{0,1}/*))`
+    #[test]
+    fn test_bip49_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            BIP49Public(pubkey, fingerprint, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt",
+                "2NCTQfJ1sZa3wQ3pPseYRHbaNEpC3AquEfX",
+                "2MveFxAuC8BYPzTybx7FxSzW8HSd8ATT4z7",
+            ],
+        );
+        check(
+            BIP49Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "2NF2vttKibwyxigxtx95Zw8K7JhDbo5zPVJ",
+                "2Mtmyd8taksxNVWCJ4wVvaiss7QPZGcAJuH",
+                "2NBs3CTVYPr1HCzjB4YFsnWCPCtNg8uMEfp",
+            ],
+        );
+    }
+
+    // BIP84 `wpkh(key/84'/0'/0'/{0,1}/*)`
+    #[test]
+    fn test_bip84_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            BIP84(prvkey, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "bcrt1qkmvk2nadgplmd57ztld8nf8v2yxkzmdvwtjf8s",
+                "bcrt1qx0v6zgfwe50m4kqc58cqzcyem7ay2sfl3gvqhp",
+                "bcrt1q4h7fq9zhxst6e69p3n882nfj649l7w9g3zccfp",
+            ],
+        );
+        check(
+            BIP84(prvkey, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "bcrt1qtrwtz00wxl69e5xex7amy4xzlxkaefg3gfdkxa",
+                "bcrt1qqqasfhxpkkf7zrxqnkr2sfhn74dgsrc3e3ky45",
+                "bcrt1qpks7n0gq74hsgsz3phn5vuazjjq0f5eqhsgyce",
+            ],
+        );
+    }
+
+    // BIP84 public `wpkh(key/{0,1}/*)`
+    #[test]
+    fn test_bip84_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            BIP84Public(pubkey, fingerprint, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "bcrt1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2prcdvd0h",
+                "bcrt1q3lncdlwq3lgcaaeyruynjnlccr0ve0kakh6ana",
+                "bcrt1qt9800y6xl3922jy3uyl0z33jh5wfpycyhcylr9",
+            ],
+        );
+        check(
+            BIP84Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "bcrt1qm6wqukenh7guu792lj2njgw9n78cmwsy8xy3z2",
+                "bcrt1q694twxtjn4nnrvnyvra769j0a23rllj5c6cgwp",
+                "bcrt1qhlac3c5ranv5w5emlnqs7wxhkxt8maelylcarp",
+            ],
+        );
+    }
+}

src/error.rs

@@ -24,64 +24,120 @@
 
 use std::fmt;
 
-use bitcoin::{Address, OutPoint};
+use crate::{descriptor, wallet, wallet::address_validator};
+use bitcoin::OutPoint;
 
 /// Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet)
 #[derive(Debug)]
 pub enum Error {
-    KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
-    MissingInputUTXO(usize),
+    /// Wrong number of bytes found when trying to convert to u32
     InvalidU32Bytes(Vec<u8>),
+    /// Generic error
     Generic(String),
+    /// This error is thrown when trying to convert Bare and Public key script to address
     ScriptDoesntHaveAddressForm,
-    SendAllMultipleOutputs,
-    NoAddressees,
+    /// Found multiple outputs when `single_recipient` option has been specified
+    SingleRecipientMultipleOutputs,
+    /// `single_recipient` option is selected but neither `drain_wallet` nor `manually_selected_only` are
+    SingleRecipientNoInputs,
+    /// Cannot build a tx without recipients
+    NoRecipients,
+    /// `manually_selected_only` option is selected but no utxo has been passed
+    NoUtxosSelected,
+    /// Output created is under the dust limit, 546 satoshis
     OutputBelowDustLimit(usize),
+    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
     InsufficientFunds,
-    InvalidAddressNetwork(Address),
+    /// Branch and bound coin selection possible attempts with sufficiently big UTXO set could grow
+    /// exponentially, thus a limit is set, and when hit, this error is thrown
+    BnBTotalTriesExceeded,
+    /// Branch and bound coin selection tries to avoid needing a change by finding the right inputs for
+    /// the desired outputs plus fee, if there is not such combination this error is thrown
+    BnBNoExactMatch,
+    /// Happens when trying to spend an UTXO that is not in the internal database
     UnknownUTXO,
-    DifferentTransactions,
+    /// Thrown when a tx is not found in the internal database
     TransactionNotFound,
+    /// Happens when trying to bump a transaction that is already confirmed
     TransactionConfirmed,
+    /// Trying to replace a tx that has a sequence >= `0xFFFFFFFE`
     IrreplaceableTransaction,
+    /// When bumping a tx the fee rate requested is lower than required
     FeeRateTooLow {
+        /// Required fee rate (satoshi/vbyte)
         required: crate::types::FeeRate,
     },
-
+    /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
+    FeeTooLow {
+        /// Required fee absolute value (satoshi)
+        required: u64,
+    },
+    /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
+    /// key in the descriptor must either be a master key itself (having depth = 0) or have an
+    /// explicit origin provided
+    ///
+    /// [`TxBuilder::add_global_xpubs`]: crate::wallet::tx_builder::TxBuilder::add_global_xpubs
+    MissingKeyOrigin(String),
+    /// Error while working with [`keys`](crate::keys)
+    Key(crate::keys::KeyError),
+    /// Descriptor checksum mismatch
     ChecksumMismatch,
-    DifferentDescriptorStructure,
-
-    SpendingPolicyRequired,
+    /// Spending policy is not compatible with this [`KeychainKind`](crate::types::KeychainKind)
+    SpendingPolicyRequired(crate::types::KeychainKind),
+    /// Error while extracting and manipulating policies
     InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
-
+    /// Signing error
     Signer(crate::wallet::signer::SignerError),
 
     // Blockchain interface errors
-    Uncapable(crate::blockchain::Capability),
+    /// Thrown when trying to call a method that requires a network connection, [`Wallet::sync`](crate::Wallet::sync) and [`Wallet::broadcast`](crate::Wallet::broadcast)
+    /// This error is thrown when creating the Client for the first time, while recovery attempts are tried
+    /// during the sync
     OfflineClient,
+    /// Progress value must be between `0.0` (included) and `100.0` (included)
     InvalidProgressValue(f32),
+    /// Progress update error (maybe the channel has been closed)
     ProgressUpdateError,
-    MissingCachedAddresses,
+    /// Requested outpoint doesn't exist in the tx (vout greater than available outputs)
     InvalidOutpoint(OutPoint),
 
+    /// Error related to the parsing and usage of descriptors
     Descriptor(crate::descriptor::error::Error),
+    /// Error that can be returned to fail the validation of an address
     AddressValidator(crate::wallet::address_validator::AddressValidatorError),
-
+    /// Encoding error
     Encode(bitcoin::consensus::encode::Error),
+    /// Miniscript error
     Miniscript(miniscript::Error),
+    /// BIP32 error
     BIP32(bitcoin::util::bip32::Error),
+    /// An ECDSA error
     Secp256k1(bitcoin::secp256k1::Error),
+    /// Error serializing or deserializing JSON data
     JSON(serde_json::Error),
+    /// Hex decoding error
     Hex(bitcoin::hashes::hex::Error),
+    /// Partially signed bitcoin transaction error
     PSBT(bitcoin::util::psbt::Error),
 
+    //KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
+    //MissingInputUTXO(usize),
+    //InvalidAddressNetwork(Address),
+    //DifferentTransactions,
+    //DifferentDescriptorStructure,
+    //Uncapable(crate::blockchain::Capability),
+    //MissingCachedAddresses,
     #[cfg(feature = "electrum")]
+    /// Electrum client error
     Electrum(electrum_client::Error),
     #[cfg(feature = "esplora")]
+    /// Esplora client error
     Esplora(crate::blockchain::esplora::EsploraError),
     #[cfg(feature = "compact_filters")]
+    /// Compact filters client error)
     CompactFilters(crate::blockchain::compact_filters::CompactFiltersError),
     #[cfg(feature = "key-value-db")]
+    /// Sled database error
     Sled(sled::Error),
 }
 
@@ -95,24 +151,32 @@ impl std::error::Error for Error {}
 
 macro_rules! impl_error {
     ( $from:ty, $to:ident ) => {
-        impl std::convert::From<$from> for Error {
+        impl_error!($from, $to, Error);
+    };
+    ( $from:ty, $to:ident, $impl_for:ty ) => {
+        impl std::convert::From<$from> for $impl_for {
             fn from(err: $from) -> Self {
-                Error::$to(err)
+                <$impl_for>::$to(err)
             }
         }
     };
 }
 
-impl_error!(crate::descriptor::error::Error, Descriptor);
-impl_error!(
-    crate::wallet::address_validator::AddressValidatorError,
-    AddressValidator
-);
-impl_error!(
-    crate::descriptor::policy::PolicyError,
-    InvalidPolicyPathError
-);
-impl_error!(crate::wallet::signer::SignerError, Signer);
+impl_error!(descriptor::error::Error, Descriptor);
+impl_error!(address_validator::AddressValidatorError, AddressValidator);
+impl_error!(descriptor::policy::PolicyError, InvalidPolicyPathError);
+impl_error!(wallet::signer::SignerError, Signer);
+
+impl From<crate::keys::KeyError> for Error {
+    fn from(key_error: crate::keys::KeyError) -> Error {
+        match key_error {
+            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
+            crate::keys::KeyError::BIP32(inner) => Error::BIP32(inner),
+            crate::keys::KeyError::InvalidChecksum => Error::ChecksumMismatch,
+            e => Error::Key(e),
+        }
+    }
+}
 
 impl_error!(bitcoin::consensus::encode::Error, Encode);
 impl_error!(miniscript::Error, Miniscript);

src/keys/bip39.rs

@@ -0,0 +1,173 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! BIP-0039
+
+// TODO: maybe write our own implementation of bip39? Seems stupid to have an extra dependency for
+// something that should be fairly simple to re-implement.
+
+use bitcoin::util::bip32;
+use bitcoin::Network;
+
+use miniscript::ScriptContext;
+
+use bip39::{Language, Mnemonic, MnemonicType, Seed};
+
+use super::{any_network, DerivableKey, DescriptorKey, GeneratableKey, GeneratedKey, KeyError};
+
+/// Type for a BIP39 mnemonic with an optional passphrase
+pub type MnemonicWithPassphrase = (Mnemonic, Option<String>);
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for Seed {
+    fn add_metadata(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let xprv = bip32::ExtendedPrivKey::new_master(Network::Bitcoin, &self.as_bytes())?;
+        let descriptor_key = xprv.add_metadata(source, derivation_path)?;
+
+        // here we must choose one network to build the xpub, but since the bip39 standard doesn't
+        // encode the network, the xpub we create is actually valid everywhere. so we override the
+        // valid networks with `any_network()`.
+        Ok(descriptor_key.override_valid_networks(any_network()))
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for MnemonicWithPassphrase {
+    fn add_metadata(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let (mnemonic, passphrase) = self;
+        let seed = Seed::new(&mnemonic, passphrase.as_deref().unwrap_or(""));
+        seed.add_metadata(source, derivation_path)
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for Mnemonic {
+    fn add_metadata(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        (self, None).add_metadata(source, derivation_path)
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for Mnemonic {
+    type Entropy = [u8; 32];
+
+    type Options = (MnemonicType, Language);
+    type Error = Option<bip39::ErrorKind>;
+
+    fn generate_with_entropy(
+        (mnemonic_type, language): Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        let entropy = &entropy.as_ref()[..(mnemonic_type.entropy_bits() / 8)];
+        let mnemonic = Mnemonic::from_entropy(entropy, language).map_err(|e| e.downcast().ok())?;
+
+        Ok(GeneratedKey::new(mnemonic, any_network()))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+
+    use bitcoin::util::bip32;
+
+    use bip39::{Language, Mnemonic, MnemonicType};
+
+    use crate::keys::{any_network, GeneratableKey, GeneratedKey};
+
+    #[test]
+    fn test_keys_bip39_mnemonic() {
+        let mnemonic =
+            "aim bunker wash balance finish force paper analyst cabin spoon stable organ";
+        let mnemonic = Mnemonic::from_phrase(mnemonic, Language::English).unwrap();
+        let path = bip32::DerivationPath::from_str("m/44'/0'/0'/0").unwrap();
+
+        let key = (mnemonic, path);
+        let (desc, keys, networks) = crate::descriptor!(wpkh(key)).unwrap();
+        assert_eq!(desc.to_string(), "wpkh([be83839f/44'/0'/0']xpub6DCQ1YcqvZtSwGWMrwHELPehjWV3f2MGZ69yBADTxFEUAoLwb5Mp5GniQK6tTp3AgbngVz9zEFbBJUPVnkG7LFYt8QMTfbrNqs6FNEwAPKA/0/*)");
+        assert_eq!(keys.len(), 1);
+        assert_eq!(networks.len(), 3);
+    }
+
+    #[test]
+    fn test_keys_bip39_mnemonic_passphrase() {
+        let mnemonic =
+            "aim bunker wash balance finish force paper analyst cabin spoon stable organ";
+        let mnemonic = Mnemonic::from_phrase(mnemonic, Language::English).unwrap();
+        let path = bip32::DerivationPath::from_str("m/44'/0'/0'/0").unwrap();
+
+        let key = ((mnemonic, Some("passphrase".into())), path);
+        let (desc, keys, networks) = crate::descriptor!(wpkh(key)).unwrap();
+        assert_eq!(desc.to_string(), "wpkh([8f6cb80c/44'/0'/0']xpub6DWYS8bbihFevy29M4cbw4ZR3P5E12jB8R88gBDWCTCNpYiDHhYWNywrCF9VZQYagzPmsZpxXpytzSoxynyeFr4ZyzheVjnpLKuse4fiwZw/0/*)");
+        assert_eq!(keys.len(), 1);
+        assert_eq!(networks.len(), 3);
+    }
+
+    #[test]
+    fn test_keys_generate_bip39() {
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate_with_entropy(
+                (MnemonicType::Words12, Language::English),
+                crate::keys::test::TEST_ENTROPY,
+            )
+            .unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+        assert_eq!(
+            generated_mnemonic.to_string(),
+            "primary fetch primary fetch primary fetch primary fetch primary fetch primary fever"
+        );
+
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate_with_entropy(
+                (MnemonicType::Words24, Language::English),
+                crate::keys::test::TEST_ENTROPY,
+            )
+            .unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+        assert_eq!(generated_mnemonic.to_string(), "primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary foster");
+    }
+
+    #[test]
+    fn test_keys_generate_bip39_random() {
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate((MnemonicType::Words12, Language::English)).unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate((MnemonicType::Words24, Language::English)).unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+    }
+}

src/keys/mod.rs

@@ -0,0 +1,747 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Key formats
+
+use std::any::TypeId;
+use std::collections::HashSet;
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::str::FromStr;
+
+use bitcoin::secp256k1;
+
+use bitcoin::util::bip32;
+use bitcoin::{Network, PrivateKey, PublicKey};
+
+pub use miniscript::descriptor::{
+    DescriptorPublicKey, DescriptorSecretKey, DescriptorSinglePriv, DescriptorSinglePub,
+    SortedMultiVec,
+};
+use miniscript::descriptor::{DescriptorXKey, KeyMap};
+pub use miniscript::ScriptContext;
+use miniscript::{Miniscript, Terminal};
+
+use crate::wallet::utils::SecpCtx;
+
+#[cfg(feature = "keys-bip39")]
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+pub mod bip39;
+
+/// Set of valid networks for a key
+pub type ValidNetworks = HashSet<Network>;
+
+/// Create a set containing mainnet, testnet and regtest
+pub fn any_network() -> ValidNetworks {
+    vec![Network::Bitcoin, Network::Testnet, Network::Regtest]
+        .into_iter()
+        .collect()
+}
+/// Create a set only containing mainnet
+pub fn mainnet_network() -> ValidNetworks {
+    vec![Network::Bitcoin].into_iter().collect()
+}
+/// Create a set containing testnet and regtest
+pub fn test_networks() -> ValidNetworks {
+    vec![Network::Testnet, Network::Regtest]
+        .into_iter()
+        .collect()
+}
+/// Compute the intersection of two sets
+pub fn merge_networks(a: &ValidNetworks, b: &ValidNetworks) -> ValidNetworks {
+    a.intersection(b).cloned().collect()
+}
+
+/// Container for public or secret keys
+#[derive(Debug)]
+pub enum DescriptorKey<Ctx: ScriptContext> {
+    #[doc(hidden)]
+    Public(DescriptorPublicKey, ValidNetworks, PhantomData<Ctx>),
+    #[doc(hidden)]
+    Secret(DescriptorSecretKey, ValidNetworks, PhantomData<Ctx>),
+}
+
+impl<Ctx: ScriptContext> DescriptorKey<Ctx> {
+    /// Create an instance given a public key and a set of valid networks
+    pub fn from_public(public: DescriptorPublicKey, networks: ValidNetworks) -> Self {
+        DescriptorKey::Public(public, networks, PhantomData)
+    }
+
+    /// Create an instance given a secret key and a set of valid networks
+    pub fn from_secret(secret: DescriptorSecretKey, networks: ValidNetworks) -> Self {
+        DescriptorKey::Secret(secret, networks, PhantomData)
+    }
+
+    /// Override the computed set of valid networks
+    pub fn override_valid_networks(self, networks: ValidNetworks) -> Self {
+        match self {
+            DescriptorKey::Public(key, _, _) => DescriptorKey::Public(key, networks, PhantomData),
+            DescriptorKey::Secret(key, _, _) => DescriptorKey::Secret(key, networks, PhantomData),
+        }
+    }
+
+    // This method is used internally by `bdk::fragment!` and `bdk::descriptor!`. It has to be
+    // public because it is effectively called by external crates, once the macros are expanded,
+    // but since it is not meant to be part of the public api we hide it from the docs.
+    #[doc(hidden)]
+    pub fn extract(
+        self,
+        secp: &SecpCtx,
+    ) -> Result<(DescriptorPublicKey, KeyMap, ValidNetworks), KeyError> {
+        match self {
+            DescriptorKey::Public(public, valid_networks, _) => {
+                Ok((public, KeyMap::default(), valid_networks))
+            }
+            DescriptorKey::Secret(secret, valid_networks, _) => {
+                let mut key_map = KeyMap::with_capacity(1);
+
+                let public = secret
+                    .as_public(secp)
+                    .map_err(|e| miniscript::Error::Unexpected(e.to_string()))?;
+                key_map.insert(public.clone(), secret);
+
+                Ok((public, key_map, valid_networks))
+            }
+        }
+    }
+}
+
+/// Enum representation of the known valid [`ScriptContext`]s
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub enum ScriptContextEnum {
+    /// Legacy scripts
+    Legacy,
+    /// Segwitv0 scripts
+    Segwitv0,
+}
+
+impl ScriptContextEnum {
+    /// Returns whether the script context is [`ScriptContextEnum::Legacy`]
+    pub fn is_legacy(&self) -> bool {
+        self == &ScriptContextEnum::Legacy
+    }
+
+    /// Returns whether the script context is [`ScriptContextEnum::Segwitv0`]
+    pub fn is_segwit_v0(&self) -> bool {
+        self == &ScriptContextEnum::Segwitv0
+    }
+}
+
+/// Trait that adds extra useful methods to [`ScriptContext`]s
+pub trait ExtScriptContext: ScriptContext {
+    /// Returns the [`ScriptContext`] as a [`ScriptContextEnum`]
+    fn as_enum() -> ScriptContextEnum;
+
+    /// Returns whether the script context is [`Legacy`](miniscript::Legacy)
+    fn is_legacy() -> bool {
+        Self::as_enum().is_legacy()
+    }
+
+    /// Returns whether the script context is [`Segwitv0`](miniscript::Segwitv0)
+    fn is_segwit_v0() -> bool {
+        Self::as_enum().is_segwit_v0()
+    }
+}
+
+impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
+    fn as_enum() -> ScriptContextEnum {
+        match TypeId::of::<Ctx>() {
+            t if t == TypeId::of::<miniscript::Legacy>() => ScriptContextEnum::Legacy,
+            t if t == TypeId::of::<miniscript::Segwitv0>() => ScriptContextEnum::Segwitv0,
+            _ => unimplemented!("Unknown ScriptContext type"),
+        }
+    }
+}
+
+/// Trait for objects that can be turned into a public or secret [`DescriptorKey`]
+///
+/// The generic type `Ctx` is used to define the context in which the key is valid: some key
+/// formats, like the mnemonics used by Electrum wallets, encode internally whether the wallet is
+/// legacy or segwit. Thus, trying to turn a valid legacy mnemonic into a `DescriptorKey`
+/// that would become part of a segwit descriptor should fail.
+///
+/// For key types that do care about this, the [`ExtScriptContext`] trait provides some useful
+/// methods that can be used to check at runtime which `Ctx` is being used.
+///
+/// For key types that can do this check statically (because they can only work within a
+/// single `Ctx`), the "specialized" trait can be implemented to make the compiler handle the type
+/// checking.
+///
+/// Keys also have control over the networks they support: constructing the return object with
+/// [`DescriptorKey::from_public`] or [`DescriptorKey::from_secret`] allows to specify a set of
+/// [`ValidNetworks`].
+///
+/// ## Examples
+///
+/// Key type valid in any context:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{DescriptorKey, KeyError, ScriptContext, ToDescriptorKey};
+///
+/// pub struct MyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for MyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         self.pubkey.to_descriptor_key()
+///     }
+/// }
+/// ```
+///
+/// Key type that is only valid on mainnet:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{
+///     mainnet_network, DescriptorKey, DescriptorPublicKey, DescriptorSinglePub, KeyError,
+///     ScriptContext, ToDescriptorKey,
+/// };
+///
+/// pub struct MyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for MyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         Ok(DescriptorKey::from_public(
+///             DescriptorPublicKey::SinglePub(DescriptorSinglePub {
+///                 origin: None,
+///                 key: self.pubkey,
+///             }),
+///             mainnet_network(),
+///         ))
+///     }
+/// }
+/// ```
+///
+/// Key type that internally encodes in which context it's valid. The context is checked at runtime:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{DescriptorKey, ExtScriptContext, KeyError, ScriptContext, ToDescriptorKey};
+///
+/// pub struct MyKeyType {
+///     is_legacy: bool,
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext + 'static> ToDescriptorKey<Ctx> for MyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         if Ctx::is_legacy() == self.is_legacy {
+///             self.pubkey.to_descriptor_key()
+///         } else {
+///             Err(KeyError::InvalidScriptContext)
+///         }
+///     }
+/// }
+/// ```
+///
+/// Key type that can only work within [`miniscript::Segwitv0`] context. Only the specialized version
+/// of the trait is implemented.
+///
+/// This example deliberately fails to compile, to demonstrate how the compiler can catch when keys
+/// are misused. In this case, the "segwit-only" key is used to build a `pkh()` descriptor, which
+/// makes the compiler (correctly) fail.
+///
+/// ```compile_fail
+/// use bdk::bitcoin::PublicKey;
+/// use std::str::FromStr;
+///
+/// use bdk::keys::{DescriptorKey, KeyError, ToDescriptorKey};
+///
+/// pub struct MySegwitOnlyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl ToDescriptorKey<bdk::miniscript::Segwitv0> for MySegwitOnlyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<bdk::miniscript::Segwitv0>, KeyError> {
+///         self.pubkey.to_descriptor_key()
+///     }
+/// }
+///
+/// let key = MySegwitOnlyKeyType {
+///     pubkey: PublicKey::from_str("...")?,
+/// };
+/// let (descriptor, _, _) = bdk::descriptor!(pkh(key))?;
+/// //                                       ^^^^^ changing this to `wpkh` would make it compile
+///
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub trait ToDescriptorKey<Ctx: ScriptContext>: Sized {
+    /// Turn the key into a [`DescriptorKey`] within the requested [`ScriptContext`]
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError>;
+}
+
+/// Trait for keys that can be derived.
+///
+/// When extra metadata are provided, a [`DerivableKey`] can be transofrmed into a
+/// [`DescriptorKey`]: the trait [`ToDescriptorKey`] is automatically implemented
+/// for `(DerivableKey, DerivationPath)` and
+/// `(DerivableKey, KeySource, DerivationPath)` tuples.
+///
+/// For key types that don't encode any indication about the path to use (like bip39), it's
+/// generally recommended to implemented this trait instead of [`ToDescriptorKey`]. The same
+/// rules regarding script context and valid networks apply.
+///
+/// [`DerivationPath`]: (bip32::DerivationPath)
+pub trait DerivableKey<Ctx: ScriptContext> {
+    /// Add a extra metadata, consume `self` and turn it into a [`DescriptorKey`]
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError>;
+}
+
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPubKey {
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorPublicKey::XPub(DescriptorXKey {
+            origin,
+            xkey: self,
+            derivation_path,
+            is_wildcard: true,
+        })
+        .to_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPrivKey {
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::XPrv(DescriptorXKey {
+            origin,
+            xkey: self,
+            derivation_path,
+            is_wildcard: true,
+        })
+        .to_descriptor_key()
+    }
+}
+
+/// Output of a [`GeneratableKey`] key generation
+pub struct GeneratedKey<K, Ctx: ScriptContext> {
+    key: K,
+    valid_networks: ValidNetworks,
+    phantom: PhantomData<Ctx>,
+}
+
+impl<K, Ctx: ScriptContext> GeneratedKey<K, Ctx> {
+    fn new(key: K, valid_networks: ValidNetworks) -> Self {
+        GeneratedKey {
+            key,
+            valid_networks,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Consumes `self` and returns the key
+    pub fn into_key(self) -> K {
+        self.key
+    }
+}
+
+impl<K, Ctx: ScriptContext> Deref for GeneratedKey<K, Ctx> {
+    type Target = K;
+
+    fn deref(&self) -> &Self::Target {
+        &self.key
+    }
+}
+
+// Make generated "derivable" keys themselves "derivable". Also make sure they are assigned the
+// right `valid_networks`.
+impl<Ctx, K> DerivableKey<Ctx> for GeneratedKey<K, Ctx>
+where
+    Ctx: ScriptContext,
+    K: DerivableKey<Ctx>,
+{
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let descriptor_key = self.key.add_metadata(origin, derivation_path)?;
+        Ok(descriptor_key.override_valid_networks(self.valid_networks))
+    }
+}
+
+// Make generated keys directly usable in descriptors, and make sure they get assigned the right
+// `valid_networks`.
+impl<Ctx, K> ToDescriptorKey<Ctx> for GeneratedKey<K, Ctx>
+where
+    Ctx: ScriptContext,
+    K: ToDescriptorKey<Ctx>,
+{
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let desc_key = self.key.to_descriptor_key()?;
+        Ok(desc_key.override_valid_networks(self.valid_networks))
+    }
+}
+
+/// Trait for keys that can be generated
+///
+/// The same rules about [`ScriptContext`] and [`ValidNetworks`] from [`ToDescriptorKey`] apply.
+///
+/// This trait is particularly useful when combined with [`DerivableKey`]: if `Self`
+/// implements it, the returned [`GeneratedKey`] will also implement it. The same is true for
+/// [`ToDescriptorKey`]: the generated keys can be directly used in descriptors if `Self` is also
+/// [`ToDescriptorKey`].
+pub trait GeneratableKey<Ctx: ScriptContext>: Sized {
+    /// Type specifying the amount of entropy required e.g. [u8;32]
+    type Entropy: AsMut<[u8]> + Default;
+
+    /// Extra options required by the `generate_with_entropy`
+    type Options;
+    /// Returned error in case of failure
+    type Error: std::fmt::Debug;
+
+    /// Generate a key given the extra options and the entropy
+    fn generate_with_entropy(
+        options: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error>;
+
+    /// Generate a key given the options with a random entropy
+    fn generate(options: Self::Options) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        use rand::{thread_rng, Rng};
+
+        let mut entropy = Self::Entropy::default();
+        thread_rng().fill(entropy.as_mut());
+        Self::generate_with_entropy(options, entropy)
+    }
+}
+
+/// Trait that allows generating a key with the default options
+///
+/// This trait is automatically implemented if the [`GeneratableKey::Options`] implements [`Default`].
+pub trait GeneratableDefaultOptions<Ctx>: GeneratableKey<Ctx>
+where
+    Ctx: ScriptContext,
+    <Self as GeneratableKey<Ctx>>::Options: Default,
+{
+    /// Generate a key with the default options and a given entropy
+    fn generate_with_entropy_default(
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        Self::generate_with_entropy(Default::default(), entropy)
+    }
+
+    /// Generate a key with the default options and a random entropy
+    fn generate_default() -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        Self::generate(Default::default())
+    }
+}
+
+/// Automatic implementation of [`GeneratableDefaultOptions`] for [`GeneratableKey`]s where
+/// `Options` implements `Default`
+impl<Ctx, K> GeneratableDefaultOptions<Ctx> for K
+where
+    Ctx: ScriptContext,
+    K: GeneratableKey<Ctx>,
+    <K as GeneratableKey<Ctx>>::Options: Default,
+{
+}
+
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::ExtendedPrivKey {
+    type Entropy = [u8; 32];
+
+    type Options = ();
+    type Error = bip32::Error;
+
+    fn generate_with_entropy(
+        _: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        // pick a arbitrary network here, but say that we support all of them
+        let xprv = bip32::ExtendedPrivKey::new_master(Network::Bitcoin, entropy.as_ref())?;
+        Ok(GeneratedKey::new(xprv, any_network()))
+    }
+}
+
+/// Options for generating a [`PrivateKey`]
+///
+/// Defaults to creating compressed keys, which save on-chain bytes and fees
+#[derive(Debug, Copy, Clone)]
+pub struct PrivateKeyGenerateOptions {
+    /// Whether the generated key should be "compressed" or not
+    pub compressed: bool,
+}
+
+impl Default for PrivateKeyGenerateOptions {
+    fn default() -> Self {
+        PrivateKeyGenerateOptions { compressed: true }
+    }
+}
+
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for PrivateKey {
+    type Entropy = [u8; secp256k1::constants::SECRET_KEY_SIZE];
+
+    type Options = PrivateKeyGenerateOptions;
+    type Error = bip32::Error;
+
+    fn generate_with_entropy(
+        options: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        // pick a arbitrary network here, but say that we support all of them
+        let key = secp256k1::SecretKey::from_slice(&entropy)?;
+        let private_key = PrivateKey {
+            compressed: options.compressed,
+            network: Network::Bitcoin,
+            key,
+        };
+
+        Ok(GeneratedKey::new(private_key, any_network()))
+    }
+}
+
+impl<Ctx: ScriptContext, T: DerivableKey<Ctx>> ToDescriptorKey<Ctx> for (T, bip32::DerivationPath) {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        self.0.add_metadata(None, self.1)
+    }
+}
+
+impl<Ctx: ScriptContext, T: DerivableKey<Ctx>> ToDescriptorKey<Ctx>
+    for (T, bip32::KeySource, bip32::DerivationPath)
+{
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        self.0.add_metadata(Some(self.1), self.2)
+    }
+}
+
+fn expand_multi_keys<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<(Vec<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError> {
+    let (pks, key_maps_networks): (Vec<_>, Vec<_>) = pks
+        .into_iter()
+        .map(|key| Ok::<_, KeyError>(key.to_descriptor_key()?.extract(secp)?))
+        .collect::<Result<Vec<_>, _>>()?
+        .into_iter()
+        .map(|(a, b, c)| (a, (b, c)))
+        .unzip();
+
+    let (key_map, valid_networks) = key_maps_networks.into_iter().fold(
+        (KeyMap::default(), any_network()),
+        |(mut keys_acc, net_acc), (key, net)| {
+            keys_acc.extend(key.into_iter());
+            let net_acc = merge_networks(&net_acc, &net);
+
+            (keys_acc, net_acc)
+        },
+    );
+
+    Ok((pks, key_map, valid_networks))
+}
+
+// Used internally by `bdk::fragment!` to build `pk_k()` fragments
+#[doc(hidden)]
+pub fn make_pk<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    descriptor_key: Pk,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), KeyError> {
+    let (key, key_map, valid_networks) = descriptor_key.to_descriptor_key()?.extract(secp)?;
+
+    Ok((
+        Miniscript::from_ast(Terminal::PkK(key))?,
+        key_map,
+        valid_networks,
+    ))
+}
+
+// Used internally by `bdk::fragment!` to build `multi()` fragments
+#[doc(hidden)]
+pub fn make_multi<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    thresh: usize,
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), KeyError> {
+    let (pks, key_map, valid_networks) = expand_multi_keys(pks, secp)?;
+
+    Ok((
+        Miniscript::from_ast(Terminal::Multi(thresh, pks))?,
+        key_map,
+        valid_networks,
+    ))
+}
+
+// Used internally by `bdk::descriptor!` to build `sortedmulti()` fragments
+#[doc(hidden)]
+pub fn make_sortedmulti_inner<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    thresh: usize,
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<
+    (
+        SortedMultiVec<DescriptorPublicKey, Ctx>,
+        KeyMap,
+        ValidNetworks,
+    ),
+    KeyError,
+> {
+    let (pks, key_map, valid_networks) = expand_multi_keys(pks, secp)?;
+
+    Ok((SortedMultiVec::new(thresh, pks)?, key_map, valid_networks))
+}
+
+/// The "identity" conversion is used internally by some `bdk::fragment`s
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for DescriptorKey<Ctx> {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        Ok(self)
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for DescriptorPublicKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let networks = match self {
+            DescriptorPublicKey::SinglePub(_) => any_network(),
+            DescriptorPublicKey::XPub(DescriptorXKey { xkey, .. })
+                if xkey.network == Network::Bitcoin =>
+            {
+                mainnet_network()
+            }
+            _ => test_networks(),
+        };
+
+        Ok(DescriptorKey::from_public(self, networks))
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for PublicKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorPublicKey::SinglePub(DescriptorSinglePub {
+            key: self,
+            origin: None,
+        })
+        .to_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for DescriptorSecretKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let networks = match &self {
+            DescriptorSecretKey::SinglePriv(sk) if sk.key.network == Network::Bitcoin => {
+                mainnet_network()
+            }
+            DescriptorSecretKey::XPrv(DescriptorXKey { xkey, .. })
+                if xkey.network == Network::Bitcoin =>
+            {
+                mainnet_network()
+            }
+            _ => test_networks(),
+        };
+
+        Ok(DescriptorKey::from_secret(self, networks))
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for &'_ str {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::from_str(self)
+            .map_err(|e| KeyError::Message(e.to_string()))?
+            .to_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for PrivateKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::SinglePriv(DescriptorSinglePriv {
+            key: self,
+            origin: None,
+        })
+        .to_descriptor_key()
+    }
+}
+
+/// Errors thrown while working with [`keys`](crate::keys)
+#[derive(Debug)]
+pub enum KeyError {
+    /// The key cannot exist in the given script context
+    InvalidScriptContext,
+    /// The key is not valid for the given network
+    InvalidNetwork,
+    /// The key has an invalid checksum
+    InvalidChecksum,
+
+    /// Custom error message
+    Message(String),
+
+    /// BIP32 error
+    BIP32(bitcoin::util::bip32::Error),
+    /// Miniscript error
+    Miniscript(miniscript::Error),
+}
+
+impl_error!(miniscript::Error, Miniscript, KeyError);
+impl_error!(bitcoin::util::bip32::Error, BIP32, KeyError);
+
+impl std::fmt::Display for KeyError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for KeyError {}
+
+#[cfg(test)]
+pub mod test {
+    use bitcoin::util::bip32;
+
+    use super::*;
+
+    pub const TEST_ENTROPY: [u8; 32] = [0xAA; 32];
+
+    #[test]
+    fn test_keys_generate_xprv() {
+        let generated_xprv: GeneratedKey<_, miniscript::Segwitv0> =
+            bip32::ExtendedPrivKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+
+        assert_eq!(generated_xprv.valid_networks, any_network());
+        assert_eq!(generated_xprv.to_string(), "xprv9s21ZrQH143K4Xr1cJyqTvuL2FWR8eicgY9boWqMBv8MDVUZ65AXHnzBrK1nyomu6wdcabRgmGTaAKawvhAno1V5FowGpTLVx3jxzE5uk3Q");
+    }
+
+    #[test]
+    fn test_keys_generate_wif() {
+        let generated_wif: GeneratedKey<_, miniscript::Segwitv0> =
+            bitcoin::PrivateKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+
+        assert_eq!(generated_wif.valid_networks, any_network());
+        assert_eq!(
+            generated_wif.to_string(),
+            "L2wTu6hQrnDMiFNWA5na6jB12ErGQqtXwqpSL7aWquJaZG8Ai3ch"
+        );
+    }
+}

src/lib.rs

@@ -22,6 +22,8 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
+// rustdoc will warn if there are missing docs
+#![warn(missing_docs)]
 // only enables the `doc_cfg` feature when
 // the `docsrs` configuration attribute is defined
 #![cfg_attr(docsrs, feature(doc_cfg))]
@@ -29,6 +31,179 @@
 // `test-md-docs` is enabled
 #![cfg_attr(feature = "test-md-docs", feature(external_doc))]
 
+//! A modern, lightweight, descriptor-based wallet library written in Rust.
+//!
+//! # About
+//!
+//! The BDK library aims to be the core building block for Bitcoin wallets of any kind.
+//!
+//! * It uses [Miniscript](https://github.com/rust-bitcoin/rust-miniscript) to support descriptors with generalized conditions. This exact same library can be used to build
+//!   single-sig wallets, multisigs, timelocked contracts and more.
+//! * It supports multiple blockchain backends and databases, allowing developers to choose exactly what's right for their projects.
+//! * It is built to be cross-platform: the core logic works on desktop, mobile, and even WebAssembly.
+//! * It is very easy to extend: developers can implement customized logic for blockchain backends, databases, signers, coin selection, and more, without having to fork and modify this library.
+//!
+//! # A Tour of BDK
+//!
+//! BDK consists of a number of modules that provide a range of functionality
+//! essential for implementing descriptor based Bitcoin wallet applications in Rust. In this
+//! section, we will take a brief tour of BDK, summarizing the major APIs and
+//! their uses.
+//!
+//! The easiest way to get started is to add bdk to your dependencies with the default features.
+//! The default features include a simple key-value database ([`sled`](sled)) to cache
+//! blockchain data and an [electrum](https://docs.rs/electrum-client/) blockchain client to
+//! interact with the bitcoin P2P network.
+//!
+//! ```toml
+//! bdk = "0.2.0"
+//! ```
+//!
+//! ## Sync the balance of a descriptor
+//!
+//! ### Example
+//! ```ignore
+//! use bdk::Wallet;
+//! use bdk::database::MemoryDatabase;
+//! use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+//!
+//! use bdk::electrum_client::Client;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+//!     let wallet = Wallet::new(
+//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!         ElectrumBlockchain::from(client)
+//!     )?;
+//!
+//!     wallet.sync(noop_progress(), None)?;
+//!
+//!     println!("Descriptor balance: {} SAT", wallet.get_balance()?);
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Generate a few addresses
+//!
+//! ### Example
+//! ```
+//! use bdk::{Wallet, OfflineWallet};
+//! use bdk::database::MemoryDatabase;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let wallet: OfflineWallet<_> = Wallet::new_offline(
+//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!     )?;
+//!
+//!     println!("Address #0: {}", wallet.get_new_address()?);
+//!     println!("Address #1: {}", wallet.get_new_address()?);
+//!     println!("Address #2: {}", wallet.get_new_address()?);
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Create a transaction
+//!
+//! ### Example
+//! ```ignore
+//! use base64::decode;
+//! use bdk::{FeeRate, TxBuilder, Wallet};
+//! use bdk::database::MemoryDatabase;
+//! use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+//!
+//! use bdk::electrum_client::Client;
+//!
+//! use bitcoin::consensus::serialize;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+//!     let wallet = Wallet::new(
+//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!         ElectrumBlockchain::from(client)
+//!     )?;
+//!
+//!     wallet.sync(noop_progress(), None)?;
+//!
+//!     let send_to = wallet.get_new_address()?;
+//!     let (psbt, details) = wallet.create_tx(
+//!         TxBuilder::with_recipients(vec![(send_to.script_pubkey(), 50_000)])
+//!             .enable_rbf()
+//!             .do_not_spend_change()
+//!             .fee_rate(FeeRate::from_sat_per_vb(5.0))
+//!     )?;
+//!
+//!     println!("Transaction details: {:#?}", details);
+//!     println!("Unsigned PSBT: {}", base64::encode(&serialize(&psbt)));
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Sign a transaction
+//!
+//! ### Example
+//! ```ignore
+//! use base64::decode;
+//! use bdk::{Wallet, OfflineWallet};
+//! use bdk::database::MemoryDatabase;
+//!
+//! use bitcoin::consensus::deserialize;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let wallet: OfflineWallet<_> = Wallet::new_offline(
+//!         "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!     )?;
+//!
+//!     let psbt = "...";
+//!     let psbt = deserialize(&base64::decode(psbt).unwrap())?;
+//!
+//!     let (signed_psbt, finalized) = wallet.sign(psbt, None)?;
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # Feature flags
+//!
+//! BDK uses a set of [feature flags](https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section)
+//! to reduce the amount of compiled code by allowing projects to only enable the features they need.
+//! By default, BDK enables two internal features, `key-value-db` and `electrum`.
+//!
+//! If you are new to BDK we recommended that you use the default features which will enable
+//! basic descriptor wallet functionality. More advanced users can disable the `default` features
+//! (`--no-default-features`) and build the BDK library with only the features you need.
+
+//! Below is a list of the available feature flags and the additional functionality they provide.
+//!
+//! * `all-keys`: all features for working with bitcoin keys
+//! * `async-interface`: async functions in bdk traits
+//! * `cli-utils`: utilities for creating a command line interface wallet
+//! * `keys-bip39`: [BIP-39](https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki) mnemonic codes for generating deterministic keys
+//!
+//! ## Internal features
+//!
+//! These features do not expose any new API, but influence internal implementation aspects of
+//! BDK.
+//!
+//! * `compact_filters`: [`compact_filters`](crate::blockchain::compact_filters) client protocol for interacting with the bitcoin P2P network
+//! * `electrum`: [`electrum`](crate::blockchain::electrum) client protocol for interacting with electrum servers
+//! * `esplora`: [`esplora`](crate::blockchain::esplora) client protocol for interacting with blockstream [electrs](https://github.com/Blockstream/electrs) servers
+//! * `key-value-db`: key value [`database`](crate::database) based on [`sled`](crate::sled) for caching blockchain data
+
 pub extern crate bitcoin;
 extern crate log;
 pub extern crate miniscript;
@@ -36,13 +211,16 @@ extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
+#[cfg(feature = "keys-bip39")]
+extern crate bip39;
+
 #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
 #[macro_use]
 extern crate async_trait;
 #[macro_use]
-extern crate magical_macros;
+extern crate bdk_macros;
 
-#[cfg(any(test, feature = "compact_filters"))]
+#[cfg(feature = "compact_filters")]
 #[macro_use]
 extern crate lazy_static;
 
@@ -58,12 +236,15 @@ pub extern crate sled;
 #[cfg(feature = "cli-utils")]
 pub mod cli;
 
+#[allow(unused_imports)]
 #[cfg(test)]
 #[macro_use]
 extern crate testutils;
+#[allow(unused_imports)]
 #[cfg(test)]
 #[macro_use]
 extern crate testutils_macros;
+#[allow(unused_imports)]
 #[cfg(test)]
 #[macro_use]
 extern crate serial_test;
@@ -75,10 +256,12 @@ pub mod database;
 pub mod descriptor;
 #[cfg(feature = "test-md-docs")]
 mod doctest;
+pub mod keys;
 pub(crate) mod psbt;
 pub(crate) mod types;
 pub mod wallet;
 
+pub use descriptor::template;
 pub use descriptor::HDKeyPaths;
 pub use error::Error;
 pub use types::*;

src/types.rs

@@ -29,31 +29,30 @@ use bitcoin::hash_types::Txid;
 
 use serde::{Deserialize, Serialize};
 
-/// Types of script
+/// Types of keychains
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum ScriptType {
+pub enum KeychainKind {
+    /// External
     External = 0,
+    /// Internal, usually used for change outputs
     Internal = 1,
 }
 
-impl ScriptType {
+impl KeychainKind {
+    /// Return [`KeychainKind`] as a byte
     pub fn as_byte(&self) -> u8 {
         match self {
-            ScriptType::External => 'e' as u8,
-            ScriptType::Internal => 'i' as u8,
+            KeychainKind::External => b'e',
+            KeychainKind::Internal => b'i',
         }
     }
-
-    pub fn is_internal(&self) -> bool {
-        self == &ScriptType::Internal
-    }
 }
 
-impl AsRef<[u8]> for ScriptType {
+impl AsRef<[u8]> for KeychainKind {
     fn as_ref(&self) -> &[u8] {
         match self {
-            ScriptType::External => b"e",
-            ScriptType::Internal => b"i",
+            KeychainKind::External => b"e",
+            KeychainKind::Internal => b"i",
         }
     }
 }
@@ -94,19 +93,29 @@ impl std::default::Default for FeeRate {
 /// A wallet unspent output
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
 pub struct UTXO {
+    /// Reference to a transaction output
     pub outpoint: OutPoint,
+    /// Transaction output
     pub txout: TxOut,
-    pub is_internal: bool,
+    /// Type of keychain
+    pub keychain: KeychainKind,
 }
 
 /// A wallet transaction
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
 pub struct TransactionDetails {
+    /// Optional transaction
     pub transaction: Option<Transaction>,
+    /// Transaction id
     pub txid: Txid,
+    /// Timestamp
     pub timestamp: u64,
+    /// Received value (sats)
     pub received: u64,
+    /// Sent value (sats)
     pub sent: u64,
+    /// Fee value (sats)
     pub fees: u64,
+    /// Confirmed in block height, `None` means unconfirmed
     pub height: Option<u32>,
 }

src/wallet/address_validator.rs

@@ -42,15 +42,15 @@
 //! ```
 //! # use std::sync::Arc;
 //! # use bitcoin::*;
-//! # use magical::address_validator::*;
-//! # use magical::database::*;
-//! # use magical::*;
+//! # use bdk::address_validator::*;
+//! # use bdk::database::*;
+//! # use bdk::*;
 //! struct PrintAddressAndContinue;
 //!
 //! impl AddressValidator for PrintAddressAndContinue {
 //!     fn validate(
 //!         &self,
-//!         script_type: ScriptType,
+//!         keychain: KeychainKind,
 //!         hd_keypaths: &HDKeyPaths,
 //!         script: &Script
 //!     ) -> Result<(), AddressValidatorError> {
@@ -58,7 +58,7 @@
 //!             .as_ref()
 //!             .map(Address::to_string)
 //!             .unwrap_or(script.to_string());
-//!         println!("New address of type {:?}: {}", script_type, address);
+//!         println!("New address of type {:?}: {}", keychain, address);
 //!         println!("HD keypaths: {:#?}", hd_keypaths);
 //!
 //!         Ok(())
@@ -67,11 +67,11 @@
 //!
 //! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)";
 //! let mut wallet: OfflineWallet<_> = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
-//! wallet.add_address_validator(Arc::new(Box::new(PrintAddressAndContinue)));
+//! wallet.add_address_validator(Arc::new(PrintAddressAndContinue));
 //!
 //! let address = wallet.get_new_address()?;
 //! println!("Address: {}", address);
-//! # Ok::<(), magical::Error>(())
+//! # Ok::<(), bdk::Error>(())
 //! ```
 
 use std::fmt;
@@ -79,15 +79,20 @@ use std::fmt;
 use bitcoin::Script;
 
 use crate::descriptor::HDKeyPaths;
-use crate::types::ScriptType;
+use crate::types::KeychainKind;
 
 /// Errors that can be returned to fail the validation of an address
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum AddressValidatorError {
+    /// User rejected the address
     UserRejected,
+    /// Network connection error
     ConnectionError,
+    /// Network request timeout error
     TimeoutError,
+    /// Invalid script
     InvalidScript,
+    /// A custom error message
     Message(String),
 }
 
@@ -106,11 +111,11 @@ impl std::error::Error for AddressValidatorError {}
 /// validator will be propagated up to the original caller that triggered the address generation.
 ///
 /// For a usage example see [this module](crate::address_validator)'s documentation.
-pub trait AddressValidator {
+pub trait AddressValidator: Send + Sync {
     /// Validate or inspect an address
     fn validate(
         &self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         hd_keypaths: &HDKeyPaths,
         script: &Script,
     ) -> Result<(), AddressValidatorError>;
@@ -128,7 +133,7 @@ mod test {
     impl AddressValidator for TestValidator {
         fn validate(
             &self,
-            _script_type: ScriptType,
+            _keychain: KeychainKind,
             _hd_keypaths: &HDKeyPaths,
             _script: &bitcoin::Script,
         ) -> Result<(), AddressValidatorError> {
@@ -140,7 +145,7 @@ mod test {
     #[should_panic(expected = "InvalidScript")]
     fn test_address_validator_external() {
         let (mut wallet, _, _) = get_funded_wallet(get_test_wpkh());
-        wallet.add_address_validator(Arc::new(Box::new(TestValidator)));
+        wallet.add_address_validator(Arc::new(TestValidator));
 
         wallet.get_new_address().unwrap();
     }
@@ -149,7 +154,7 @@ mod test {
     #[should_panic(expected = "InvalidScript")]
     fn test_address_validator_internal() {
         let (mut wallet, descriptors, _) = get_funded_wallet(get_test_wpkh());
-        wallet.add_address_validator(Arc::new(Box::new(TestValidator)));
+        wallet.add_address_validator(Arc::new(TestValidator));
 
         let addr = testutils!(@external descriptors, 10);
         wallet

src/wallet/export.rs

@@ -33,9 +33,9 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use magical::database::*;
-//! # use magical::wallet::export::*;
-//! # use magical::*;
+//! # use bdk::database::*;
+//! # use bdk::wallet::export::*;
+//! # use bdk::*;
 //! let import = r#"{
 //!     "descriptor": "wpkh([c258d2e4\/84h\/1h\/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe\/0\/*)",
 //!     "blockheight":1782088,
@@ -43,16 +43,21 @@
 //! }"#;
 //!
 //! let import = WalletExport::from_str(import)?;
-//! let wallet: OfflineWallet<_> = Wallet::new_offline(&import.descriptor(), import.change_descriptor().as_deref(), Network::Testnet, MemoryDatabase::default())?;
-//! # Ok::<_, magical::Error>(())
+//! let wallet: OfflineWallet<_> = Wallet::new_offline(
+//!     &import.descriptor(),
+//!     import.change_descriptor().as_ref(),
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//! )?;
+//! # Ok::<_, bdk::Error>(())
 //! ```
 //!
 //! ### Export a `Wallet`
 //! ```
 //! # use bitcoin::*;
-//! # use magical::database::*;
-//! # use magical::wallet::export::*;
-//! # use magical::*;
+//! # use bdk::database::*;
+//! # use bdk::wallet::export::*;
+//! # use bdk::*;
 //! let wallet: OfflineWallet<_> = Wallet::new_offline(
 //!     "wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/0/*)",
 //!     Some("wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/1/*)"),
@@ -61,19 +66,19 @@
 //! )?;
 //! let export = WalletExport::export_wallet(&wallet, "exported wallet", true)
 //!     .map_err(ToString::to_string)
-//!     .map_err(magical::Error::Generic)?;
+//!     .map_err(bdk::Error::Generic)?;
 //!
 //! println!("Exported: {}", export.to_string());
-//! # Ok::<_, magical::Error>(())
+//! # Ok::<_, bdk::Error>(())
 //! ```
 
 use std::str::FromStr;
 
 use serde::{Deserialize, Serialize};
 
-use miniscript::{Descriptor, ScriptContext, Terminal};
+use miniscript::{Descriptor, DescriptorPublicKey, ScriptContext, Terminal};
 
-use crate::blockchain::Blockchain;
+use crate::blockchain::BlockchainMarker;
 use crate::database::BatchDatabase;
 use crate::wallet::Wallet;
 
@@ -115,14 +120,14 @@ impl WalletExport {
     ///
     /// If the database is empty or `include_blockheight` is false, the `blockheight` field
     /// returned will be `0`.
-    pub fn export_wallet<B: Blockchain, D: BatchDatabase>(
+    pub fn export_wallet<B: BlockchainMarker, D: BatchDatabase>(
         wallet: &Wallet<B, D>,
         label: &str,
         include_blockheight: bool,
     ) -> Result<Self, &'static str> {
         let descriptor = wallet
             .descriptor
-            .to_string_with_secret(&wallet.signers.as_key_map());
+            .to_string_with_secret(&wallet.signers.as_key_map(wallet.secp_ctx()));
         Self::is_compatible_with_core(&descriptor)?;
 
         let blockheight = match wallet.database.borrow().iter_txs(false) {
@@ -133,7 +138,7 @@ impl WalletExport {
                     .into_iter()
                     .map(|tx| tx.height.unwrap_or(0))
                     .collect::<Vec<_>>();
-                heights.sort();
+                heights.sort_unstable();
 
                 *heights.last().unwrap_or(&0)
             }
@@ -145,12 +150,10 @@ impl WalletExport {
             blockheight,
         };
 
-        if export.change_descriptor()
-            != wallet
-                .change_descriptor
-                .as_ref()
-                .map(|d| d.to_string_with_secret(&wallet.change_signers.as_key_map()))
-        {
+        let desc_to_string = |d: &Descriptor<DescriptorPublicKey>| {
+            d.to_string_with_secret(&wallet.change_signers.as_key_map(wallet.secp_ctx()))
+        };
+        if export.change_descriptor() != wallet.change_descriptor.as_ref().map(desc_to_string) {
             return Err("Incompatible change descriptor");
         }
 
@@ -234,7 +237,7 @@ mod test {
         let wallet: OfflineWallet<_> = Wallet::new_offline(
             descriptor,
             Some(change_descriptor),
-            Network::Testnet,
+            Network::Bitcoin,
             get_test_db(),
         )
         .unwrap();
@@ -256,7 +259,7 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
 
         let wallet: OfflineWallet<_> =
-            Wallet::new_offline(descriptor, None, Network::Testnet, get_test_db()).unwrap();
+            Wallet::new_offline(descriptor, None, Network::Bitcoin, get_test_db()).unwrap();
         WalletExport::export_wallet(&wallet, "Test Label", true).unwrap();
     }
 
@@ -272,7 +275,7 @@ mod test {
         let wallet: OfflineWallet<_> = Wallet::new_offline(
             descriptor,
             Some(change_descriptor),
-            Network::Testnet,
+            Network::Bitcoin,
             get_test_db(),
         )
         .unwrap();
@@ -315,7 +318,7 @@ mod test {
         let wallet: OfflineWallet<_> = Wallet::new_offline(
             descriptor,
             Some(change_descriptor),
-            Network::Testnet,
+            Network::Bitcoin,
             get_test_db(),
         )
         .unwrap();

src/wallet/rbf.rs

@@ -1,127 +0,0 @@
-// Magical Bitcoin Library
-// Written in 2020 by
-//     Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020 Magical Bitcoin
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-use crate::database::Database;
-use crate::error::Error;
-use crate::types::*;
-
-/// Filters unspent utxos
-pub(super) fn filter_available<I: Iterator<Item = UTXO>, D: Database>(
-    database: &D,
-    iter: I,
-) -> Result<Vec<UTXO>, Error> {
-    Ok(iter
-        .map(|utxo| {
-            Ok(match database.get_tx(&utxo.outpoint.txid, true)? {
-                None => None,
-                Some(tx) if tx.height.is_none() => None,
-                Some(_) => Some(utxo),
-            })
-        })
-        .collect::<Result<Vec<_>, Error>>()?
-        .into_iter()
-        .filter_map(|x| x)
-        .collect())
-}
-
-#[cfg(test)]
-mod test {
-    use std::str::FromStr;
-
-    use bitcoin::{OutPoint, Transaction, TxIn, TxOut, Txid};
-
-    use super::*;
-    use crate::database::{BatchOperations, MemoryDatabase};
-
-    fn add_transaction(
-        database: &mut MemoryDatabase,
-        spend: Vec<OutPoint>,
-        outputs: Vec<u64>,
-    ) -> Txid {
-        let tx = Transaction {
-            version: 1,
-            lock_time: 0,
-            input: spend
-                .iter()
-                .cloned()
-                .map(|previous_output| TxIn {
-                    previous_output,
-                    ..Default::default()
-                })
-                .collect(),
-            output: outputs
-                .iter()
-                .cloned()
-                .map(|value| TxOut {
-                    value,
-                    ..Default::default()
-                })
-                .collect(),
-        };
-        let txid = tx.txid();
-
-        for input in &spend {
-            database.del_utxo(input).unwrap();
-        }
-        for vout in 0..outputs.len() {
-            database
-                .set_utxo(&UTXO {
-                    txout: tx.output[vout].clone(),
-                    outpoint: OutPoint {
-                        txid,
-                        vout: vout as u32,
-                    },
-                    is_internal: true,
-                })
-                .unwrap();
-        }
-        database
-            .set_tx(&TransactionDetails {
-                txid,
-                transaction: Some(tx),
-                height: None,
-                ..Default::default()
-            })
-            .unwrap();
-
-        txid
-    }
-
-    #[test]
-    fn test_filter_available() {
-        let mut database = MemoryDatabase::new();
-        add_transaction(
-            &mut database,
-            vec![OutPoint::from_str(
-                "aad194c72fd5cfd16d23da9462930ca91e35df1cfee05242b62f4034f50c3d41:5",
-            )
-            .unwrap()],
-            vec![50_000],
-        );
-
-        let filtered =
-            filter_available(&database, database.iter_utxos().unwrap().into_iter()).unwrap();
-        assert_eq!(filtered, &[]);
-    }
-}

src/wallet/signer.rs

@@ -30,12 +30,13 @@
 //! ```
 //! # use std::sync::Arc;
 //! # use std::str::FromStr;
+//! # use bitcoin::secp256k1::{Secp256k1, All};
 //! # use bitcoin::*;
 //! # use bitcoin::util::psbt;
 //! # use bitcoin::util::bip32::Fingerprint;
-//! # use magical::signer::*;
-//! # use magical::database::*;
-//! # use magical::*;
+//! # use bdk::signer::*;
+//! # use bdk::database::*;
+//! # use bdk::*;
 //! # #[derive(Debug)]
 //! # struct CustomHSM;
 //! # impl CustomHSM {
@@ -62,6 +63,7 @@
 //!         &self,
 //!         psbt: &mut psbt::PartiallySignedTransaction,
 //!         input_index: Option<usize>,
+//!         _secp: &Secp256k1<All>,
 //!     ) -> Result<(), SignerError> {
 //!         let input_index = input_index.ok_or(SignerError::InputIndexOutOfRange)?;
 //!         self.device.sign_input(psbt, input_index)?;
@@ -79,13 +81,13 @@
 //! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)";
 //! let mut wallet: OfflineWallet<_> = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
 //! wallet.add_signer(
-//!     ScriptType::External,
+//!     KeychainKind::External,
 //!     Fingerprint::from_str("e30f11b8").unwrap().into(),
 //!     SignerOrdering(200),
-//!     Arc::new(Box::new(custom_signer))
+//!     Arc::new(custom_signer)
 //! );
 //!
-//! # Ok::<_, magical::Error>(())
+//! # Ok::<_, bdk::Error>(())
 //! ```
 
 use std::cmp::Ordering;
@@ -100,29 +102,32 @@ use bitcoin::hashes::{hash160, Hash};
 use bitcoin::secp256k1::{Message, Secp256k1};
 use bitcoin::util::bip32::{ExtendedPrivKey, Fingerprint};
 use bitcoin::util::{bip143, psbt};
-use bitcoin::{PrivateKey, SigHash, SigHashType};
+use bitcoin::{PrivateKey, Script, SigHash, SigHashType};
 
-use miniscript::descriptor::{DescriptorPublicKey, DescriptorSecretKey, DescriptorXKey, KeyMap};
+use miniscript::descriptor::{DescriptorSecretKey, DescriptorSinglePriv, DescriptorXKey, KeyMap};
 use miniscript::{Legacy, MiniscriptKey, Segwitv0};
 
+use super::utils::SecpCtx;
 use crate::descriptor::XKeyUtils;
 
 /// Identifier of a signer in the `SignersContainers`. Used as a key to find the right signer among
 /// multiple of them
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum SignerId<Pk: MiniscriptKey> {
-    PkHash(<Pk as MiniscriptKey>::Hash),
+#[derive(Debug, Clone, Ord, PartialOrd, PartialEq, Eq, Hash)]
+pub enum SignerId {
+    /// Bitcoin HASH160 (RIPEMD160 after SHA256) hash of an ECDSA public key
+    PkHash(hash160::Hash),
+    /// The fingerprint of a BIP32 extended key
     Fingerprint(Fingerprint),
 }
 
-impl From<hash160::Hash> for SignerId<DescriptorPublicKey> {
-    fn from(hash: hash160::Hash) -> SignerId<DescriptorPublicKey> {
+impl From<hash160::Hash> for SignerId {
+    fn from(hash: hash160::Hash) -> SignerId {
         SignerId::PkHash(hash)
     }
 }
 
-impl From<Fingerprint> for SignerId<DescriptorPublicKey> {
-    fn from(fing: Fingerprint) -> SignerId<DescriptorPublicKey> {
+impl From<Fingerprint> for SignerId {
+    fn from(fing: Fingerprint) -> SignerId {
         SignerId::Fingerprint(fing)
     }
 }
@@ -132,10 +137,10 @@ impl From<Fingerprint> for SignerId<DescriptorPublicKey> {
 pub enum SignerError {
     /// The private key is missing for the required public key
     MissingKey,
+    /// The private key in use has the right fingerprint but derives differently than expected
+    InvalidKey,
     /// The user canceled the operation
     UserCanceled,
-    /// The sighash is missing in the PSBT input
-    MissingSighash,
     /// Input index is out of range
     InputIndexOutOfRange,
     /// The `non_witness_utxo` field of the transaction is required to sign this input
@@ -162,7 +167,7 @@ impl std::error::Error for SignerError {}
 ///
 /// This trait can be implemented to provide customized signers to the wallet. For an example see
 /// [`this module`](crate::wallet::signer)'s documentation.
-pub trait Signer: fmt::Debug {
+pub trait Signer: fmt::Debug + Send + Sync {
     /// Sign a PSBT
     ///
     /// The `input_index` argument is only provided if the wallet doesn't declare to sign the whole
@@ -172,6 +177,7 @@ pub trait Signer: fmt::Debug {
         &self,
         psbt: &mut psbt::PartiallySignedTransaction,
         input_index: Option<usize>,
+        secp: &SecpCtx,
     ) -> Result<(), SignerError>;
 
     /// Return whether or not the signer signs the whole transaction in one go instead of every
@@ -193,26 +199,35 @@ impl Signer for DescriptorXKey<ExtendedPrivKey> {
         &self,
         psbt: &mut psbt::PartiallySignedTransaction,
         input_index: Option<usize>,
+        secp: &SecpCtx,
     ) -> Result<(), SignerError> {
         let input_index = input_index.unwrap();
         if input_index >= psbt.inputs.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
-        let deriv_path = match psbt.inputs[input_index]
+        let (public_key, deriv_path) = match psbt.inputs[input_index]
             .hd_keypaths
             .iter()
-            .filter_map(|(_, &(fingerprint, ref path))| self.matches(fingerprint.clone(), &path))
+            .filter_map(|(pk, &(fingerprint, ref path))| {
+                if self.matches(&(fingerprint, path.clone()), &secp).is_some() {
+                    Some((pk, path))
+                } else {
+                    None
+                }
+            })
             .next()
         {
-            Some(deriv_path) => deriv_path,
-            None => return Ok(()), // TODO: should report an error maybe?
+            Some((pk, full_path)) => (pk, full_path.clone()),
+            None => return Ok(()),
         };
 
-        let ctx = Secp256k1::signing_only();
-
-        let derived_key = self.xkey.derive_priv(&ctx, &deriv_path).unwrap();
-        derived_key.private_key.sign(psbt, Some(input_index))
+        let derived_key = self.xkey.derive_priv(&secp, &deriv_path).unwrap();
+        if &derived_key.private_key.public_key(&secp) != public_key {
+            Err(SignerError::InvalidKey)
+        } else {
+            derived_key.private_key.sign(psbt, Some(input_index), secp)
+        }
     }
 
     fn sign_whole_tx(&self) -> bool {
@@ -229,15 +244,14 @@ impl Signer for PrivateKey {
         &self,
         psbt: &mut psbt::PartiallySignedTransaction,
         input_index: Option<usize>,
+        secp: &SecpCtx,
     ) -> Result<(), SignerError> {
         let input_index = input_index.unwrap();
         if input_index >= psbt.inputs.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
 
-        let ctx = Secp256k1::signing_only();
-
-        let pubkey = self.public_key(&ctx);
+        let pubkey = self.public_key(&secp);
         if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
             return Ok(());
         }
@@ -251,7 +265,7 @@ impl Signer for PrivateKey {
             None => Legacy::sighash(psbt, input_index)?,
         };
 
-        let signature = ctx.sign(
+        let signature = secp.sign(
             &Message::from_slice(&hash.into_inner()[..]).unwrap(),
             &self.key,
         );
@@ -272,7 +286,10 @@ impl Signer for PrivateKey {
     }
 
     fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
-        Some(DescriptorSecretKey::PrivKey(self.clone()))
+        Some(DescriptorSecretKey::SinglePriv(DescriptorSinglePriv {
+            key: *self,
+            origin: None,
+        }))
     }
 }
 
@@ -291,13 +308,13 @@ impl std::default::Default for SignerOrdering {
 }
 
 #[derive(Debug, Clone)]
-struct SignersContainerKey<Pk: MiniscriptKey> {
-    id: SignerId<Pk>,
+struct SignersContainerKey {
+    id: SignerId,
     ordering: SignerOrdering,
 }
 
-impl<Pk: MiniscriptKey> From<(SignerId<Pk>, SignerOrdering)> for SignersContainerKey<Pk> {
-    fn from(tuple: (SignerId<Pk>, SignerOrdering)) -> Self {
+impl From<(SignerId, SignerOrdering)> for SignersContainerKey {
+    fn from(tuple: (SignerId, SignerOrdering)) -> Self {
         SignersContainerKey {
             id: tuple.0,
             ordering: tuple.1,
@@ -307,39 +324,40 @@ impl<Pk: MiniscriptKey> From<(SignerId<Pk>, SignerOrdering)> for SignersContaine
 
 /// Container for multiple signers
 #[derive(Debug, Default, Clone)]
-pub struct SignersContainer<Pk: MiniscriptKey>(
-    BTreeMap<SignersContainerKey<Pk>, Arc<Box<dyn Signer>>>,
-);
+pub struct SignersContainer(BTreeMap<SignersContainerKey, Arc<dyn Signer>>);
 
-impl SignersContainer<DescriptorPublicKey> {
-    pub fn as_key_map(&self) -> KeyMap {
+impl SignersContainer {
+    /// Create a map of public keys to secret keys
+    pub fn as_key_map(&self, secp: &SecpCtx) -> KeyMap {
         self.0
             .values()
             .filter_map(|signer| signer.descriptor_secret_key())
-            .filter_map(|secret| secret.as_public().ok().map(|public| (public, secret)))
+            .filter_map(|secret| secret.as_public(secp).ok().map(|public| (public, secret)))
             .collect()
     }
 }
 
-impl From<KeyMap> for SignersContainer<DescriptorPublicKey> {
-    fn from(keymap: KeyMap) -> SignersContainer<DescriptorPublicKey> {
+impl From<KeyMap> for SignersContainer {
+    fn from(keymap: KeyMap) -> SignersContainer {
+        let secp = Secp256k1::new();
         let mut container = SignersContainer::new();
 
         for (_, secret) in keymap {
             match secret {
-                DescriptorSecretKey::PrivKey(private_key) => container.add_external(
+                DescriptorSecretKey::SinglePriv(private_key) => container.add_external(
                     SignerId::from(
                         private_key
+                            .key
                             .public_key(&Secp256k1::signing_only())
                             .to_pubkeyhash(),
                     ),
                     SignerOrdering::default(),
-                    Arc::new(Box::new(private_key)),
+                    Arc::new(private_key.key),
                 ),
                 DescriptorSecretKey::XPrv(xprv) => container.add_external(
-                    SignerId::from(xprv.root_fingerprint()),
+                    SignerId::from(xprv.root_fingerprint(&secp)),
                     SignerOrdering::default(),
-                    Arc::new(Box::new(xprv)),
+                    Arc::new(xprv),
                 ),
             };
         }
@@ -348,34 +366,30 @@ impl From<KeyMap> for SignersContainer<DescriptorPublicKey> {
     }
 }
 
-impl<Pk: MiniscriptKey> SignersContainer<Pk> {
+impl SignersContainer {
     /// Default constructor
     pub fn new() -> Self {
         SignersContainer(Default::default())
     }
 
     /// Adds an external signer to the container for the specified id. Optionally returns the
-    /// signer that was previosuly in the container, if any
+    /// signer that was previously in the container, if any
     pub fn add_external(
         &mut self,
-        id: SignerId<Pk>,
+        id: SignerId,
         ordering: SignerOrdering,
-        signer: Arc<Box<dyn Signer>>,
-    ) -> Option<Arc<Box<dyn Signer>>> {
+        signer: Arc<dyn Signer>,
+    ) -> Option<Arc<dyn Signer>> {
         self.0.insert((id, ordering).into(), signer)
     }
 
     /// Removes a signer from the container and returns it
-    pub fn remove(
-        &mut self,
-        id: SignerId<Pk>,
-        ordering: SignerOrdering,
-    ) -> Option<Arc<Box<dyn Signer>>> {
+    pub fn remove(&mut self, id: SignerId, ordering: SignerOrdering) -> Option<Arc<dyn Signer>> {
         self.0.remove(&(id, ordering).into())
     }
 
     /// Returns the list of identifiers of all the signers in the container
-    pub fn ids(&self) -> Vec<&SignerId<Pk>> {
+    pub fn ids(&self) -> Vec<&SignerId> {
         self.0
             .keys()
             .map(|SignersContainerKey { id, .. }| id)
@@ -383,19 +397,20 @@ impl<Pk: MiniscriptKey> SignersContainer<Pk> {
     }
 
     /// Returns the list of signers in the container, sorted by lowest to highest `ordering`
-    pub fn signers(&self) -> Vec<&Arc<Box<dyn Signer>>> {
+    pub fn signers(&self) -> Vec<&Arc<dyn Signer>> {
         self.0.values().collect()
     }
 
     /// Finds the signer with lowest ordering for a given id in the container.
-    pub fn find(&self, id: SignerId<Pk>) -> Option<&Arc<Box<dyn Signer>>> {
+    pub fn find(&self, id: SignerId) -> Option<&Arc<dyn Signer>> {
         self.0
             .range((
                 Included(&(id.clone(), SignerOrdering(0)).into()),
-                Included(&(id, SignerOrdering(usize::MAX)).into()),
+                Included(&(id.clone(), SignerOrdering(usize::MAX)).into()),
             ))
+            .filter(|(k, _)| k.id == id)
             .map(|(_, v)| v)
-            .nth(0)
+            .next()
     }
 }
 
@@ -418,10 +433,10 @@ impl ComputeSighash for Legacy {
         let psbt_input = &psbt.inputs[input_index];
         let tx_input = &psbt.global.unsigned_tx.input[input_index];
 
-        let sighash = psbt_input.sighash_type.ok_or(SignerError::MissingSighash)?;
-        let script = match &psbt_input.redeem_script {
-            &Some(ref redeem_script) => redeem_script.clone(),
-            &None => {
+        let sighash = psbt_input.sighash_type.unwrap_or(SigHashType::All);
+        let script = match psbt_input.redeem_script {
+            Some(ref redeem_script) => redeem_script.clone(),
+            None => {
                 let non_witness_utxo = psbt_input
                     .non_witness_utxo
                     .as_ref()
@@ -444,6 +459,16 @@ impl ComputeSighash for Legacy {
     }
 }
 
+fn p2wpkh_script_code(script: &Script) -> Script {
+    ScriptBuilder::new()
+        .push_opcode(opcodes::all::OP_DUP)
+        .push_opcode(opcodes::all::OP_HASH160)
+        .push_slice(&script[2..])
+        .push_opcode(opcodes::all::OP_EQUALVERIFY)
+        .push_opcode(opcodes::all::OP_CHECKSIG)
+        .into_script()
+}
+
 impl ComputeSighash for Segwitv0 {
     fn sighash(
         psbt: &psbt::PartiallySignedTransaction,
@@ -455,7 +480,7 @@ impl ComputeSighash for Segwitv0 {
 
         let psbt_input = &psbt.inputs[input_index];
 
-        let sighash = psbt_input.sighash_type.ok_or(SignerError::MissingSighash)?;
+        let sighash = psbt_input.sighash_type.unwrap_or(SigHashType::All);
 
         let witness_utxo = psbt_input
             .witness_utxo
@@ -463,17 +488,18 @@ impl ComputeSighash for Segwitv0 {
             .ok_or(SignerError::MissingNonWitnessUtxo)?;
         let value = witness_utxo.value;
 
-        let script = match &psbt_input.witness_script {
-            &Some(ref witness_script) => witness_script.clone(),
-            &None => {
+        let script = match psbt_input.witness_script {
+            Some(ref witness_script) => witness_script.clone(),
+            None => {
                 if witness_utxo.script_pubkey.is_v0_p2wpkh() {
-                    ScriptBuilder::new()
-                        .push_opcode(opcodes::all::OP_DUP)
-                        .push_opcode(opcodes::all::OP_HASH160)
-                        .push_slice(&witness_utxo.script_pubkey[2..])
-                        .push_opcode(opcodes::all::OP_EQUALVERIFY)
-                        .push_opcode(opcodes::all::OP_CHECKSIG)
-                        .into_script()
+                    p2wpkh_script_code(&witness_utxo.script_pubkey)
+                } else if psbt_input
+                    .redeem_script
+                    .as_ref()
+                    .map(Script::is_v0_p2wpkh)
+                    .unwrap_or(false)
+                {
+                    p2wpkh_script_code(&psbt_input.redeem_script.as_ref().unwrap())
                 } else {
                     return Err(SignerError::MissingWitnessScript);
                 }
@@ -492,22 +518,159 @@ impl ComputeSighash for Segwitv0 {
     }
 }
 
-impl<Pk: MiniscriptKey> PartialOrd for SignersContainerKey<Pk> {
+impl PartialOrd for SignersContainerKey {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<Pk: MiniscriptKey> Ord for SignersContainerKey<Pk> {
+impl Ord for SignersContainerKey {
     fn cmp(&self, other: &Self) -> Ordering {
-        self.ordering.cmp(&other.ordering)
+        self.ordering
+            .cmp(&other.ordering)
+            .then(self.id.cmp(&other.id))
     }
 }
 
-impl<Pk: MiniscriptKey> PartialEq for SignersContainerKey<Pk> {
+impl PartialEq for SignersContainerKey {
     fn eq(&self, other: &Self) -> bool {
-        self.ordering == other.ordering
+        self.id == other.id && self.ordering == other.ordering
     }
 }
 
-impl<Pk: MiniscriptKey> Eq for SignersContainerKey<Pk> {}
+impl Eq for SignersContainerKey {}
+
+#[cfg(test)]
+mod signers_container_tests {
+    use super::*;
+    use crate::descriptor;
+    use crate::descriptor::ToWalletDescriptor;
+    use crate::keys::{DescriptorKey, ToDescriptorKey};
+    use bitcoin::secp256k1::All;
+    use bitcoin::util::bip32;
+    use bitcoin::util::psbt::PartiallySignedTransaction;
+    use bitcoin::Network;
+    use miniscript::ScriptContext;
+    use std::str::FromStr;
+
+    // Signers added with the same ordering (like `Ordering::default`) created from `KeyMap`
+    // should be preserved and not overwritten.
+    // This happens usually when a set of signers is created from a descriptor with private keys.
+    #[test]
+    fn signers_with_same_ordering() {
+        let (prvkey1, _, _) = setup_keys(TPRV0_STR);
+        let (prvkey2, _, _) = setup_keys(TPRV1_STR);
+        let desc = descriptor!(sh(multi(2, prvkey1, prvkey2))).unwrap();
+        let (_, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+
+        let signers = SignersContainer::from(keymap);
+        assert_eq!(signers.ids().len(), 2);
+
+        let signers = signers.signers();
+        assert_eq!(signers.len(), 2);
+    }
+
+    #[test]
+    fn signers_sorted_by_ordering() {
+        let mut signers = SignersContainer::new();
+        let signer1 = Arc::new(DummySigner);
+        let signer2 = Arc::new(DummySigner);
+        let signer3 = Arc::new(DummySigner);
+
+        signers.add_external(
+            SignerId::Fingerprint(b"cafe"[..].into()),
+            SignerOrdering(1),
+            signer1.clone(),
+        );
+        signers.add_external(
+            SignerId::Fingerprint(b"babe"[..].into()),
+            SignerOrdering(2),
+            signer2.clone(),
+        );
+        signers.add_external(
+            SignerId::Fingerprint(b"feed"[..].into()),
+            SignerOrdering(3),
+            signer3.clone(),
+        );
+
+        // Check that signers are sorted from lowest to highest ordering
+        let signers = signers.signers();
+        assert_eq!(Arc::as_ptr(signers[0]), Arc::as_ptr(&signer1));
+        assert_eq!(Arc::as_ptr(signers[1]), Arc::as_ptr(&signer2));
+        assert_eq!(Arc::as_ptr(signers[2]), Arc::as_ptr(&signer3));
+    }
+
+    #[test]
+    fn find_signer_by_id() {
+        let mut signers = SignersContainer::new();
+        let signer1: Arc<dyn Signer> = Arc::new(DummySigner);
+        let signer2: Arc<dyn Signer> = Arc::new(DummySigner);
+        let signer3: Arc<dyn Signer> = Arc::new(DummySigner);
+        let signer4: Arc<dyn Signer> = Arc::new(DummySigner);
+
+        let id1 = SignerId::Fingerprint(b"cafe"[..].into());
+        let id2 = SignerId::Fingerprint(b"babe"[..].into());
+        let id3 = SignerId::Fingerprint(b"feed"[..].into());
+        let id_nonexistent = SignerId::Fingerprint(b"fefe"[..].into());
+
+        signers.add_external(id1.clone(), SignerOrdering(1), signer1.clone());
+        signers.add_external(id2.clone(), SignerOrdering(2), signer2.clone());
+        signers.add_external(id3.clone(), SignerOrdering(3), signer3.clone());
+
+        assert!(
+            matches!(signers.find(id1), Some(signer) if Arc::as_ptr(&signer1) == Arc::as_ptr(signer))
+        );
+        assert!(
+            matches!(signers.find(id2), Some(signer) if Arc::as_ptr(&signer2) == Arc::as_ptr(signer))
+        );
+        assert!(
+            matches!(signers.find(id3.clone()), Some(signer) if Arc::as_ptr(&signer3) == Arc::as_ptr(signer))
+        );
+
+        // The `signer4` has the same ID as `signer3` but lower ordering.
+        // It should be found by `id3` instead of `signer3`.
+        signers.add_external(id3.clone(), SignerOrdering(2), signer4.clone());
+        assert!(
+            matches!(signers.find(id3), Some(signer) if Arc::as_ptr(&signer4) == Arc::as_ptr(signer))
+        );
+
+        // Can't find anything with ID that doesn't exist
+        assert!(matches!(signers.find(id_nonexistent), None));
+    }
+
+    #[derive(Debug)]
+    struct DummySigner;
+    impl Signer for DummySigner {
+        fn sign(
+            &self,
+            _psbt: &mut PartiallySignedTransaction,
+            _input_index: Option<usize>,
+            _secp: &SecpCtx,
+        ) -> Result<(), SignerError> {
+            Ok(())
+        }
+
+        fn sign_whole_tx(&self) -> bool {
+            true
+        }
+    }
+
+    const TPRV0_STR:&str = "tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf";
+    const TPRV1_STR:&str = "tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N";
+
+    const PATH: &str = "m/44'/1'/0'/0";
+
+    fn setup_keys<Ctx: ScriptContext>(
+        tprv: &str,
+    ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
+        let secp: Secp256k1<All> = Secp256k1::new();
+        let path = bip32::DerivationPath::from_str(PATH).unwrap();
+        let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
+        let tpub = bip32::ExtendedPubKey::from_private(&secp, &tprv);
+        let fingerprint = tprv.fingerprint(&secp);
+        let prvkey = (tprv, path.clone()).to_descriptor_key().unwrap();
+        let pubkey = (tpub, path).to_descriptor_key().unwrap();
+
+        (prvkey, pubkey, fingerprint)
+    }
+}

src/wallet/tx_builder.rs

@@ -29,7 +29,8 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use magical::*;
+//! # use bdk::*;
+//! # use bdk::wallet::tx_builder::CreateTx;
 //! # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
 //! // Create a transaction with one output to `to_address` of 50_000 satoshi, with a custom fee rate
 //! // of 5.0 satoshi/vbyte, only spending non-change outputs and with RBF signaling
@@ -38,88 +39,196 @@
 //!     .fee_rate(FeeRate::from_sat_per_vb(5.0))
 //!     .do_not_spend_change()
 //!     .enable_rbf();
+//! # let builder: TxBuilder<bdk::database::MemoryDatabase, _, CreateTx> = builder;
 //! ```
 
 use std::collections::BTreeMap;
+use std::collections::HashSet;
 use std::default::Default;
+use std::marker::PhantomData;
 
 use bitcoin::{OutPoint, Script, SigHashType, Transaction};
 
 use super::coin_selection::{CoinSelectionAlgorithm, DefaultCoinSelectionAlgorithm};
-use crate::types::{FeeRate, UTXO};
+use crate::database::Database;
+use crate::types::{FeeRate, KeychainKind, UTXO};
+
+/// Context in which the [`TxBuilder`] is valid
+pub trait TxBuilderContext: std::fmt::Debug + Default + Clone {}
+
+/// [`Wallet::create_tx`](super::Wallet::create_tx) context
+#[derive(Debug, Default, Clone)]
+pub struct CreateTx;
+impl TxBuilderContext for CreateTx {}
+
+/// [`Wallet::bump_fee`](super::Wallet::bump_fee) context
+#[derive(Debug, Default, Clone)]
+pub struct BumpFee;
+impl TxBuilderContext for BumpFee {}
 
 /// A transaction builder
 ///
 /// This structure contains the configuration that the wallet must follow to build a transaction.
 ///
 /// For an example see [this module](super::tx_builder)'s documentation;
-#[derive(Debug, Default)]
-pub struct TxBuilder<Cs: CoinSelectionAlgorithm> {
+#[derive(Debug)]
+pub struct TxBuilder<D: Database, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext> {
     pub(crate) recipients: Vec<(Script, u64)>,
-    pub(crate) send_all: bool,
-    pub(crate) fee_rate: Option<FeeRate>,
-    pub(crate) policy_path: Option<BTreeMap<String, Vec<usize>>>,
-    pub(crate) utxos: Option<Vec<OutPoint>>,
-    pub(crate) unspendable: Option<Vec<OutPoint>>,
+    pub(crate) drain_wallet: bool,
+    pub(crate) single_recipient: Option<Script>,
+    pub(crate) fee_policy: Option<FeePolicy>,
+    pub(crate) internal_policy_path: Option<BTreeMap<String, Vec<usize>>>,
+    pub(crate) external_policy_path: Option<BTreeMap<String, Vec<usize>>>,
+    pub(crate) utxos: Vec<OutPoint>,
+    pub(crate) unspendable: HashSet<OutPoint>,
+    pub(crate) manually_selected_only: bool,
     pub(crate) sighash: Option<SigHashType>,
     pub(crate) ordering: TxOrdering,
     pub(crate) locktime: Option<u32>,
-    pub(crate) rbf: Option<u32>,
+    pub(crate) rbf: Option<RBFValue>,
     pub(crate) version: Option<Version>,
     pub(crate) change_policy: ChangeSpendPolicy,
     pub(crate) force_non_witness_utxo: bool,
+    pub(crate) add_global_xpubs: bool,
     pub(crate) coin_selection: Cs,
+    pub(crate) include_output_redeem_witness_script: bool,
+
+    phantom: PhantomData<(D, Ctx)>,
 }
 
-impl TxBuilder<DefaultCoinSelectionAlgorithm> {
+#[derive(Debug)]
+pub(crate) enum FeePolicy {
+    FeeRate(FeeRate),
+    FeeAmount(u64),
+}
+
+impl std::default::Default for FeePolicy {
+    fn default() -> Self {
+        FeePolicy::FeeRate(FeeRate::default_min_relay_fee())
+    }
+}
+
+// Unfortunately derive doesn't work with `PhantomData`: https://github.com/rust-lang/rust/issues/26925
+impl<D: Database, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext> Default
+    for TxBuilder<D, Cs, Ctx>
+where
+    Cs: Default,
+{
+    fn default() -> Self {
+        TxBuilder {
+            recipients: Default::default(),
+            drain_wallet: Default::default(),
+            single_recipient: Default::default(),
+            fee_policy: Default::default(),
+            internal_policy_path: Default::default(),
+            external_policy_path: Default::default(),
+            utxos: Default::default(),
+            unspendable: Default::default(),
+            manually_selected_only: Default::default(),
+            sighash: Default::default(),
+            ordering: Default::default(),
+            locktime: Default::default(),
+            rbf: Default::default(),
+            version: Default::default(),
+            change_policy: Default::default(),
+            force_non_witness_utxo: Default::default(),
+            add_global_xpubs: Default::default(),
+            coin_selection: Default::default(),
+            include_output_redeem_witness_script: Default::default(),
+
+            phantom: PhantomData,
+        }
+    }
+}
+
+// methods supported by both contexts, but only for `DefaultCoinSelectionAlgorithm`
+impl<D: Database, Ctx: TxBuilderContext> TxBuilder<D, DefaultCoinSelectionAlgorithm, Ctx> {
     /// Create an empty builder
     pub fn new() -> Self {
         Self::default()
     }
-
-    /// Create a builder starting from a list of recipients
-    pub fn with_recipients(recipients: Vec<(Script, u64)>) -> Self {
-        Self::default().set_recipients(recipients)
-    }
 }
 
-impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
-    /// Replace the recipients already added with a new list
-    pub fn set_recipients(mut self, recipients: Vec<(Script, u64)>) -> Self {
-        self.recipients = recipients;
-        self
-    }
-
-    /// Add a recipient to the internal list
-    pub fn add_recipient(mut self, script_pubkey: Script, amount: u64) -> Self {
-        self.recipients.push((script_pubkey, amount));
-        self
-    }
-
-    /// Send all the selected utxos to a single output
-    ///
-    /// Adding more than one recipients with this option enabled will result in an error.
-    ///
-    /// The value associated with the only recipient is irrelevant and will be replaced by the wallet.
-    pub fn send_all(mut self) -> Self {
-        self.send_all = true;
-        self
-    }
-
+// methods supported by both contexts, for any CoinSelectionAlgorithm
+impl<D: Database, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext> TxBuilder<D, Cs, Ctx> {
     /// Set a custom fee rate
     pub fn fee_rate(mut self, fee_rate: FeeRate) -> Self {
-        self.fee_rate = Some(fee_rate);
+        self.fee_policy = Some(FeePolicy::FeeRate(fee_rate));
         self
     }
 
-    /// Set the policy path to use while creating the transaction
+    /// Set an absolute fee
+    pub fn fee_absolute(mut self, fee_amount: u64) -> Self {
+        self.fee_policy = Some(FeePolicy::FeeAmount(fee_amount));
+        self
+    }
+
+    /// Set the policy path to use while creating the transaction for a given keychain.
     ///
     /// This method accepts a map where the key is the policy node id (see
     /// [`Policy::id`](crate::descriptor::Policy::id)) and the value is the list of the indexes of
     /// the items that are intended to be satisfied from the policy node (see
     /// [`SatisfiableItem::Thresh::items`](crate::descriptor::policy::SatisfiableItem::Thresh::items)).
-    pub fn policy_path(mut self, policy_path: BTreeMap<String, Vec<usize>>) -> Self {
-        self.policy_path = Some(policy_path);
+    ///
+    /// ## Example
+    ///
+    /// An example of when the policy path is needed is the following descriptor:
+    /// `wsh(thresh(2,pk(A),sj:and_v(v:pk(B),n:older(6)),snj:and_v(v:pk(C),after(630000))))`,
+    /// derived from the miniscript policy `thresh(2,pk(A),and(pk(B),older(6)),and(pk(C),after(630000)))`.
+    /// It declares three descriptor fragments, and at the top level it uses `thresh()` to
+    /// ensure that at least two of them are satisfied. The individual fragments are:
+    ///
+    /// 1. `pk(A)`
+    /// 2. `and(pk(B),older(6))`
+    /// 3. `and(pk(C),after(630000))`
+    ///
+    /// When those conditions are combined in pairs, it's clear that the transaction needs to be created
+    /// differently depending on how the user intends to satisfy the policy afterwards:
+    ///
+    /// * If fragments `1` and `2` are used, the transaction will need to use a specific
+    ///   `n_sequence` in order to spend an `OP_CSV` branch.
+    /// * If fragments `1` and `3` are used, the transaction will need to use a specific `locktime`
+    ///   in order to spend an `OP_CLTV` branch.
+    /// * If fragments `2` and `3` are used, the transaction will need both.
+    ///
+    /// When the spending policy is represented as a tree (see
+    /// [`Wallet::policies`](super::Wallet::policies)), every node
+    /// is assigned a unique identifier that can be used in the policy path to specify which of
+    /// the node's children the user intends to satisfy: for instance, assuming the `thresh()`
+    /// root node of this example has an id of `aabbccdd`, the policy path map would look like:
+    ///
+    /// `{ "aabbccdd" => [0, 1] }`
+    ///
+    /// where the key is the node's id, and the value is a list of the children that should be
+    /// used, in no particular order.
+    ///
+    /// If a particularly complex descriptor has multiple ambiguous thresholds in its structure,
+    /// multiple entries can be added to the map, one for each node that requires an explicit path.
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// # use std::collections::BTreeMap;
+    /// # use bitcoin::*;
+    /// # use bdk::*;
+    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// let mut path = BTreeMap::new();
+    /// path.insert("aabbccdd".to_string(), vec![0, 1]);
+    ///
+    /// let builder = TxBuilder::with_recipients(vec![(to_address.script_pubkey(), 50_000)])
+    ///     .policy_path(path, KeychainKind::External);
+    /// # let builder: TxBuilder<bdk::database::MemoryDatabase, _, _> = builder;
+    /// ```
+    pub fn policy_path(
+        mut self,
+        policy_path: BTreeMap<String, Vec<usize>>,
+        keychain: KeychainKind,
+    ) -> Self {
+        let to_update = match keychain {
+            KeychainKind::Internal => &mut self.internal_policy_path,
+            KeychainKind::External => &mut self.external_policy_path,
+        };
+
+        *to_update = Some(policy_path);
         self
     }
 
@@ -128,7 +237,7 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
     /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
     /// the "utxos" and the "unspendable" list, it will be spent.
     pub fn utxos(mut self, utxos: Vec<OutPoint>) -> Self {
-        self.utxos = Some(utxos);
+        self.utxos = utxos;
         self
     }
 
@@ -137,7 +246,19 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
     /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
     /// the "utxos" and the "unspendable" list, it will be spent.
     pub fn add_utxo(mut self, utxo: OutPoint) -> Self {
-        self.utxos.get_or_insert(vec![]).push(utxo);
+        self.utxos.push(utxo);
+        self
+    }
+
+    /// Only spend utxos added by [`add_utxo`] and [`utxos`].
+    ///
+    /// The wallet will **not** add additional utxos to the transaction even if they are needed to
+    /// make the transaction valid.
+    ///
+    /// [`add_utxo`]: Self::add_utxo
+    /// [`utxos`]: Self::utxos
+    pub fn manually_selected_only(mut self) -> Self {
+        self.manually_selected_only = true;
         self
     }
 
@@ -147,7 +268,7 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
     /// [`TxBuilder::add_utxo`] have priority over these. See the docs of the two linked methods
     /// for more details.
     pub fn unspendable(mut self, unspendable: Vec<OutPoint>) -> Self {
-        self.unspendable = Some(unspendable);
+        self.unspendable = unspendable.into_iter().collect();
         self
     }
 
@@ -157,7 +278,7 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
     /// [`TxBuilder::add_utxo`] have priority over this. See the docs of the two linked methods
     /// for more details.
     pub fn add_unspendable(mut self, unspendable: OutPoint) -> Self {
-        self.unspendable.get_or_insert(vec![]).push(unspendable);
+        self.unspendable.insert(unspendable);
         self
     }
 
@@ -183,30 +304,11 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
         self
     }
 
-    /// Enable signaling RBF
-    ///
-    /// This will use the default nSequence value of `0xFFFFFFFD`.
-    pub fn enable_rbf(self) -> Self {
-        self.enable_rbf_with_sequence(0xFFFFFFFD)
-    }
-
-    /// Enable signaling RBF with a specific nSequence value
-    ///
-    /// This can cause conflicts if the wallet's descriptors contain an "older" (OP_CSV) operator
-    /// and the given `nsequence` is lower than the CSV value.
-    ///
-    /// If the `nsequence` is higher than `0xFFFFFFFD` an error will be thrown, since it would not
-    /// be a valid nSequence to signal RBF.
-    pub fn enable_rbf_with_sequence(mut self, nsequence: u32) -> Self {
-        self.rbf = Some(nsequence);
-        self
-    }
-
     /// Build a transaction with a specific version
     ///
     /// The `version` should always be greater than `0` and greater than `1` if the wallet's
     /// descriptors contain an "older" (OP_CSV) operator.
-    pub fn version(mut self, version: u32) -> Self {
+    pub fn version(mut self, version: i32) -> Self {
         self.version = Some(Version(version));
         self
     }
@@ -245,17 +347,48 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
         self
     }
 
+    /// Fill-in the [`psbt::Output::redeem_script`](bitcoin::util::psbt::Output::redeem_script) and
+    /// [`psbt::Output::witness_script`](bitcoin::util::psbt::Output::witness_script) fields.
+    ///
+    /// This is useful for signers which always require it, like ColdCard hardware wallets.
+    pub fn include_output_redeem_witness_script(mut self) -> Self {
+        self.include_output_redeem_witness_script = true;
+        self
+    }
+
+    /// Fill-in the `PSBT_GLOBAL_XPUB` field with the extended keys contained in both the external
+    /// and internal descriptors
+    ///
+    /// This is useful for offline signers that take part to a multisig. Some hardware wallets like
+    /// BitBox and ColdCard are known to require this.
+    pub fn add_global_xpubs(mut self) -> Self {
+        self.add_global_xpubs = true;
+        self
+    }
+
+    /// Spend all the available inputs. This respects filters like [`TxBuilder::unspendable`] and the change policy.
+    pub fn drain_wallet(mut self) -> Self {
+        self.drain_wallet = true;
+        self
+    }
+
     /// Choose the coin selection algorithm
     ///
     /// Overrides the [`DefaultCoinSelectionAlgorithm`](super::coin_selection::DefaultCoinSelectionAlgorithm).
-    pub fn coin_selection<P: CoinSelectionAlgorithm>(self, coin_selection: P) -> TxBuilder<P> {
+    pub fn coin_selection<P: CoinSelectionAlgorithm<D>>(
+        self,
+        coin_selection: P,
+    ) -> TxBuilder<D, P, Ctx> {
         TxBuilder {
             recipients: self.recipients,
-            send_all: self.send_all,
-            fee_rate: self.fee_rate,
-            policy_path: self.policy_path,
+            drain_wallet: self.drain_wallet,
+            single_recipient: self.single_recipient,
+            fee_policy: self.fee_policy,
+            internal_policy_path: self.internal_policy_path,
+            external_policy_path: self.external_policy_path,
             utxos: self.utxos,
             unspendable: self.unspendable,
+            manually_selected_only: self.manually_selected_only,
             sighash: self.sighash,
             ordering: self.ordering,
             locktime: self.locktime,
@@ -263,11 +396,100 @@ impl<Cs: CoinSelectionAlgorithm> TxBuilder<Cs> {
             version: self.version,
             change_policy: self.change_policy,
             force_non_witness_utxo: self.force_non_witness_utxo,
+            add_global_xpubs: self.add_global_xpubs,
+            include_output_redeem_witness_script: self.include_output_redeem_witness_script,
             coin_selection,
+
+            phantom: PhantomData,
         }
     }
 }
 
+// methods supported only by create_tx, and only for `DefaultCoinSelectionAlgorithm`
+impl<D: Database> TxBuilder<D, DefaultCoinSelectionAlgorithm, CreateTx> {
+    /// Create a builder starting from a list of recipients
+    pub fn with_recipients(recipients: Vec<(Script, u64)>) -> Self {
+        Self::default().set_recipients(recipients)
+    }
+}
+
+// methods supported only by create_tx, for any `CoinSelectionAlgorithm`
+impl<D: Database, Cs: CoinSelectionAlgorithm<D>> TxBuilder<D, Cs, CreateTx> {
+    /// Replace the recipients already added with a new list
+    pub fn set_recipients(mut self, recipients: Vec<(Script, u64)>) -> Self {
+        self.recipients = recipients;
+        self
+    }
+
+    /// Add a recipient to the internal list
+    pub fn add_recipient(mut self, script_pubkey: Script, amount: u64) -> Self {
+        self.recipients.push((script_pubkey, amount));
+        self
+    }
+
+    /// Set a single recipient that will get all the selected funds minus the fee. No change will
+    /// be created
+    ///
+    /// This method overrides any recipient set with [`set_recipients`](Self::set_recipients) or
+    /// [`add_recipient`](Self::add_recipient).
+    ///
+    /// It can only be used in conjunction with [`drain_wallet`](Self::drain_wallet) to send the
+    /// entire content of the wallet (minus filters) to a single recipient or with a
+    /// list of manually selected UTXOs by enabling [`manually_selected_only`](Self::manually_selected_only)
+    /// and selecting them with [`utxos`](Self::utxos) or [`add_utxo`](Self::add_utxo).
+    ///
+    /// When bumping the fees of a transaction made with this option, the user should remeber to
+    /// add [`maintain_single_recipient`](Self::maintain_single_recipient) to correctly update the
+    /// single output instead of adding one more for the change.
+    pub fn set_single_recipient(mut self, recipient: Script) -> Self {
+        self.single_recipient = Some(recipient);
+        self.recipients.clear();
+
+        self
+    }
+
+    /// Enable signaling RBF
+    ///
+    /// This will use the default nSequence value of `0xFFFFFFFD`.
+    pub fn enable_rbf(mut self) -> Self {
+        self.rbf = Some(RBFValue::Default);
+        self
+    }
+
+    /// Enable signaling RBF with a specific nSequence value
+    ///
+    /// This can cause conflicts if the wallet's descriptors contain an "older" (OP_CSV) operator
+    /// and the given `nsequence` is lower than the CSV value.
+    ///
+    /// If the `nsequence` is higher than `0xFFFFFFFD` an error will be thrown, since it would not
+    /// be a valid nSequence to signal RBF.
+    pub fn enable_rbf_with_sequence(mut self, nsequence: u32) -> Self {
+        self.rbf = Some(RBFValue::Value(nsequence));
+        self
+    }
+}
+
+// methods supported only by bump_fee
+impl<D: Database> TxBuilder<D, DefaultCoinSelectionAlgorithm, BumpFee> {
+    /// Bump the fees of a transaction made with [`set_single_recipient`](Self::set_single_recipient)
+    ///
+    /// Unless extra inputs are specified with [`add_utxo`] or [`utxos`], this flag will make
+    /// `bump_fee` reduce the value of the existing output, or fail if it would be consumed
+    /// entirely given the higher new fee rate.
+    ///
+    /// If extra inputs are added and they are not entirely consumed in fees, a change output will not
+    /// be added; the existing output will simply grow in value.
+    ///
+    /// Fails if the transaction has more than one outputs.
+    ///
+    /// [`add_utxo`]: Self::add_utxo
+    /// [`utxos`]: Self::utxos
+    pub fn maintain_single_recipient(mut self) -> Self {
+        self.single_recipient = Some(Script::default());
+        self
+    }
+}
+
 /// Ordering of the transaction's inputs and outputs
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum TxOrdering {
@@ -286,6 +508,7 @@ impl Default for TxOrdering {
 }
 
 impl TxOrdering {
+    /// Sort transaction inputs and outputs by [`TxOrdering`] variant
     pub fn sort_tx(&self, tx: &mut Transaction) {
         match self {
             TxOrdering::Untouched => {}
@@ -316,7 +539,7 @@ impl TxOrdering {
 ///
 /// Has a default value of `1`
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
-pub(crate) struct Version(pub(crate) u32);
+pub(crate) struct Version(pub(crate) i32);
 
 impl Default for Version {
     fn default() -> Self {
@@ -324,6 +547,24 @@ impl Default for Version {
     }
 }
 
+/// RBF nSequence value
+///
+/// Has a default value of `0xFFFFFFFD`
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+pub(crate) enum RBFValue {
+    Default,
+    Value(u32),
+}
+
+impl RBFValue {
+    pub(crate) fn get_value(&self) -> u32 {
+        match self {
+            RBFValue::Default => 0xFFFFFFFD,
+            RBFValue::Value(v) => *v,
+        }
+    }
+}
+
 /// Policy regarding the use of change outputs when creating a transaction
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum ChangeSpendPolicy {
@@ -342,11 +583,11 @@ impl Default for ChangeSpendPolicy {
 }
 
 impl ChangeSpendPolicy {
-    pub(crate) fn filter_utxos<I: Iterator<Item = UTXO>>(&self, iter: I) -> Vec<UTXO> {
+    pub(crate) fn is_satisfied_by(&self, utxo: &UTXO) -> bool {
         match self {
-            ChangeSpendPolicy::ChangeAllowed => iter.collect(),
-            ChangeSpendPolicy::OnlyChange => iter.filter(|utxo| utxo.is_internal).collect(),
-            ChangeSpendPolicy::ChangeForbidden => iter.filter(|utxo| !utxo.is_internal).collect(),
+            ChangeSpendPolicy::ChangeAllowed => true,
+            ChangeSpendPolicy::OnlyChange => utxo.keychain == KeychainKind::Internal,
+            ChangeSpendPolicy::ChangeForbidden => utxo.keychain == KeychainKind::External,
         }
     }
 }
@@ -441,7 +682,7 @@ mod test {
                     vout: 0,
                 },
                 txout: Default::default(),
-                is_internal: false,
+                keychain: KeychainKind::External,
             },
             UTXO {
                 outpoint: OutPoint {
@@ -449,7 +690,7 @@ mod test {
                     vout: 1,
                 },
                 txout: Default::default(),
-                is_internal: true,
+                keychain: KeychainKind::Internal,
             },
         ]
     }
@@ -457,7 +698,10 @@ mod test {
     #[test]
     fn test_change_spend_policy_default() {
         let change_spend_policy = ChangeSpendPolicy::default();
-        let filtered = change_spend_policy.filter_utxos(get_test_utxos().into_iter());
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
 
         assert_eq!(filtered.len(), 2);
     }
@@ -465,19 +709,25 @@ mod test {
     #[test]
     fn test_change_spend_policy_no_internal() {
         let change_spend_policy = ChangeSpendPolicy::ChangeForbidden;
-        let filtered = change_spend_policy.filter_utxos(get_test_utxos().into_iter());
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
 
         assert_eq!(filtered.len(), 1);
-        assert_eq!(filtered[0].is_internal, false);
+        assert_eq!(filtered[0].keychain, KeychainKind::External);
     }
 
     #[test]
     fn test_change_spend_policy_only_internal() {
         let change_spend_policy = ChangeSpendPolicy::OnlyChange;
-        let filtered = change_spend_policy.filter_utxos(get_test_utxos().into_iter());
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
 
         assert_eq!(filtered.len(), 1);
-        assert_eq!(filtered[0].is_internal, true);
+        assert_eq!(filtered[0].keychain, KeychainKind::Internal);
     }
 
     #[test]

src/wallet/utils.rs

@@ -22,11 +22,27 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-use miniscript::{MiniscriptKey, Satisfier};
+use bitcoin::secp256k1::{All, Secp256k1};
+use bitcoin::util::bip32;
+
+use miniscript::descriptor::DescriptorPublicKeyCtx;
+use miniscript::{MiniscriptKey, Satisfier, ToPublicKey};
 
 // De-facto standard "dust limit" (even though it should change based on the output type)
 const DUST_LIMIT_SATOSHI: u64 = 546;
 
+// MSB of the nSequence. If set there's no consensus-constraint, so it must be disabled when
+// spending using CSV in order to enforce CSV rules
+pub(crate) const SEQUENCE_LOCKTIME_DISABLE_FLAG: u32 = 1 << 31;
+// When nSequence is lower than this flag the timelock is interpreted as block-height-based,
+// otherwise it's time-based
+pub(crate) const SEQUENCE_LOCKTIME_TYPE_FLAG: u32 = 1 << 22;
+// Mask for the bits used to express the timelock
+pub(crate) const SEQUENCE_LOCKTIME_MASK: u32 = 0x0000FFFF;
+
+// Threshold for nLockTime to be considered a block-height-based timelock rather than time-based
+pub(crate) const BLOCKS_TIMELOCK_THRESHOLD: u32 = 500000000;
+
 /// Trait to check if a value is below the dust limit
 // we implement this trait to make sure we don't mess up the comparison with off-by-one like a <
 // instead of a <= etc. The constant value for the dust limit is not public on purpose, to
@@ -56,7 +72,45 @@ impl After {
     }
 }
 
-impl<Pk: MiniscriptKey> Satisfier<Pk> for After {
+pub(crate) fn check_nsequence_rbf(rbf: u32, csv: u32) -> bool {
+    // This flag cannot be set in the nSequence when spending using OP_CSV
+    if rbf & SEQUENCE_LOCKTIME_DISABLE_FLAG != 0 {
+        return false;
+    }
+
+    let mask = SEQUENCE_LOCKTIME_TYPE_FLAG | SEQUENCE_LOCKTIME_MASK;
+    let rbf = rbf & mask;
+    let csv = csv & mask;
+
+    // Both values should be represented in the same unit (either time-based or
+    // block-height based)
+    if (rbf < SEQUENCE_LOCKTIME_TYPE_FLAG) != (csv < SEQUENCE_LOCKTIME_TYPE_FLAG) {
+        return false;
+    }
+
+    // The value should be at least `csv`
+    if rbf < csv {
+        return false;
+    }
+
+    true
+}
+
+pub(crate) fn check_nlocktime(nlocktime: u32, required: u32) -> bool {
+    // Both values should be expressed in the same unit
+    if (nlocktime < BLOCKS_TIMELOCK_THRESHOLD) != (required < BLOCKS_TIMELOCK_THRESHOLD) {
+        return false;
+    }
+
+    // The value should be at least `required`
+    if nlocktime < required {
+        return false;
+    }
+
+    true
+}
+
+impl<ToPkCtx: Copy, Pk: MiniscriptKey + ToPublicKey<ToPkCtx>> Satisfier<ToPkCtx, Pk> for After {
     fn check_after(&self, n: u32) -> bool {
         if let Some(current_height) = self.current_height {
             current_height >= n
@@ -86,7 +140,7 @@ impl Older {
     }
 }
 
-impl<Pk: MiniscriptKey> Satisfier<Pk> for Older {
+impl<ToPkCtx: Copy, Pk: MiniscriptKey + ToPublicKey<ToPkCtx>> Satisfier<ToPkCtx, Pk> for Older {
     fn check_older(&self, n: u32) -> bool {
         if let Some(current_height) = self.current_height {
             // TODO: test >= / >
@@ -97,12 +151,21 @@ impl<Pk: MiniscriptKey> Satisfier<Pk> for Older {
     }
 }
 
+pub(crate) type SecpCtx = Secp256k1<All>;
+pub(crate) fn descriptor_to_pk_ctx(secp: &SecpCtx) -> DescriptorPublicKeyCtx<'_, All> {
+    // Create a `to_pk_ctx` with a dummy derivation index, since we always use this on descriptor
+    // that have already been derived with `Descriptor::derive()`, so the child number added here
+    // is ignored.
+    DescriptorPublicKeyCtx::new(secp, bip32::ChildNumber::Normal { index: 0 })
+}
+
 pub struct ChunksIterator<I: Iterator> {
     iter: I,
     size: usize,
 }
 
 impl<I: Iterator> ChunksIterator<I> {
+    #[allow(dead_code)]
     pub fn new(iter: I, size: usize) -> Self {
         ChunksIterator { iter, size }
     }
@@ -132,6 +195,10 @@ impl<I: Iterator> Iterator for ChunksIterator<I> {
 
 #[cfg(test)]
 mod test {
+    use super::{
+        check_nlocktime, check_nsequence_rbf, BLOCKS_TIMELOCK_THRESHOLD,
+        SEQUENCE_LOCKTIME_TYPE_FLAG,
+    };
     use crate::types::FeeRate;
 
     #[test]
@@ -151,4 +218,70 @@ mod test {
         let fee = FeeRate::default_min_relay_fee();
         assert!((fee.as_sat_vb() - 1.0).abs() < 0.0001);
     }
+
+    #[test]
+    fn test_check_nsequence_rbf_msb_set() {
+        let result = check_nsequence_rbf(0x80000000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_lt_csv() {
+        let result = check_nsequence_rbf(4000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_different_unit() {
+        let result = check_nsequence_rbf(SEQUENCE_LOCKTIME_TYPE_FLAG + 5000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_mask() {
+        let result = check_nsequence_rbf(0x3f + 10_000, 5000);
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_same_unit_blocks() {
+        let result = check_nsequence_rbf(10_000, 5000);
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_same_unit_time() {
+        let result = check_nsequence_rbf(
+            SEQUENCE_LOCKTIME_TYPE_FLAG + 10_000,
+            SEQUENCE_LOCKTIME_TYPE_FLAG + 5000,
+        );
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nlocktime_lt_cltv() {
+        let result = check_nlocktime(4000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nlocktime_different_unit() {
+        let result = check_nlocktime(BLOCKS_TIMELOCK_THRESHOLD + 5000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nlocktime_same_unit_blocks() {
+        let result = check_nlocktime(10_000, 5000);
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nlocktime_same_unit_time() {
+        let result = check_nlocktime(
+            BLOCKS_TIMELOCK_THRESHOLD + 10_000,
+            BLOCKS_TIMELOCK_THRESHOLD + 5000,
+        );
+        assert_eq!(result, true);
+    }
 }

testutils-macros/Cargo.toml

@@ -1,8 +1,14 @@
 [package]
-name = "magical-testutils-macros"
-version = "0.1.0-beta.1"
+name = "bdk-testutils-macros"
+version = "0.2.0"
 authors = ["Alekos Filini <alekos.filini@gmail.com>"]
 edition = "2018"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk-testutils-macros"
+description = "Supporting testing macros for `bdk`"
+keywords = ["bdk"]
+license-file = "../LICENSE"
 
 [lib]
 proc-macro = true
@@ -11,7 +17,7 @@ name = "testutils_macros"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-syn = { version = "1.0", features = ["parsing"] }
+syn = { version = "1.0", features = ["parsing", "full"] }
 proc-macro2 = "1.0"
 quote = "1.0"
 

testutils-macros/src/lib.rs

@@ -31,7 +31,7 @@ use syn::spanned::Spanned;
 use syn::{parse, parse2, Ident, ReturnType};
 
 #[proc_macro_attribute]
-pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenStream {
+pub fn bdk_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenStream {
     let root_ident = if !attr.is_empty() {
         match parse::<syn::ExprPath>(attr) {
             Ok(parsed) => parsed,
@@ -44,12 +44,12 @@ pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenSt
             }
         }
     } else {
-        parse2::<syn::ExprPath>(quote! { magical }).unwrap()
+        parse2::<syn::ExprPath>(quote! { bdk }).unwrap()
     };
 
     match parse::<syn::ItemFn>(item) {
         Err(_) => (quote! {
-            compile_error!("#[magical_blockchain_tests] can only be used on `fn`s")
+            compile_error!("#[bdk_blockchain_tests] can only be used on `fn`s")
         })
         .into(),
         Ok(parsed) => {
@@ -63,7 +63,7 @@ pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenSt
                 ReturnType::Type(_, ref t) => t.clone(),
                 ReturnType::Default => {
                     return (quote! {
-                        compile_error!("The tagged function must return a type that impl `OnlineBlockchain`")
+                        compile_error!("The tagged function must return a type that impl `Blockchain`")
                     }).into();
                 }
             };
@@ -79,10 +79,10 @@ pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenSt
 
                 use testutils::{TestClient, serial};
 
-                use #root_ident::blockchain::{OnlineBlockchain, noop_progress};
+                use #root_ident::blockchain::{Blockchain, noop_progress};
                 use #root_ident::descriptor::ExtendedDescriptor;
                 use #root_ident::database::MemoryDatabase;
-                use #root_ident::types::ScriptType;
+                use #root_ident::types::KeychainKind;
                 use #root_ident::{Wallet, TxBuilder, FeeRate};
 
                 use super::*;
@@ -92,7 +92,7 @@ pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenSt
                 }
 
                 fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<#return_type, MemoryDatabase> {
-                    Wallet::new(&descriptors.0.to_string(), descriptors.1.as_deref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
+                    Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
                 }
 
                 fn init_single_sig() -> (Wallet<#return_type, MemoryDatabase>, (String, Option<String>), TestClient) {
@@ -120,7 +120,7 @@ pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenSt
                     wallet.sync(noop_progress(), None).unwrap();
 
                     assert_eq!(wallet.get_balance().unwrap(), 50_000);
-                    assert_eq!(wallet.list_unspent().unwrap()[0].is_internal, false);
+                    assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External);
 
                     let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
                     assert_eq!(list_tx_item.txid, txid);
@@ -520,6 +520,21 @@ pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenSt
                     assert_eq!(wallet.get_balance().unwrap(), 0);
                     assert_eq!(new_details.received, 0);
                 }
+
+                #[test]
+                #[serial]
+                fn test_sync_receive_coinbase() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let wallet_addr = wallet.get_new_address().unwrap();
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 0);
+
+                    test_client.generate(1, Some(wallet_addr));
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert!(wallet.get_balance().unwrap() > 0);
+                }
             }
 
                         };

testutils/Cargo.toml

@@ -1,25 +1,25 @@
 [package]
-name = "magical-testutils"
-version = "0.1.0-beta.1"
+name = "bdk-testutils"
+version = "0.2.0"
 authors = ["Alekos Filini <alekos.filini@gmail.com>"]
 edition = "2018"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk-testutils"
+description = "Supporting testing utilities for `bdk`"
+keywords = ["bdk"]
+license-file = "../LICENSE"
 
 [lib]
 name = "testutils"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-# The latest bitcoincore-rpc depends on an older version of bitcoin, which in turns depends on an
-# older version of secp256k1, which causes conflicts during linking. Use my fork right now, we can
-# switch back to crates.io as soon as rust-bitcoin is updated in rust-bitcoincore-rpc.
-#
-# Tracking issue: https://github.com/rust-bitcoin/rust-bitcoincore-rpc/pull/80
-
 [dependencies]
 log = "0.4.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serial_test = "0.4"
-bitcoin = "0.23"
-bitcoincore-rpc = "0.11"
-electrum-client = "0.2.0-beta.1"
+bitcoin = "0.25"
+bitcoincore-rpc = "0.12"
+electrum-client = "0.4.0-beta.1"

testutils/src/lib.rs

@@ -24,8 +24,6 @@
 
 #[macro_use]
 extern crate serde_json;
-#[macro_use]
-extern crate serial_test;
 
 pub use serial_test::serial;
 
@@ -118,16 +116,20 @@ impl TestIncomingTx {
 #[macro_export]
 macro_rules! testutils {
     ( @external $descriptors:expr, $child:expr ) => ({
-        use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
+        use bitcoin::secp256k1::Secp256k1;
+        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorPublicKeyCtx};
 
-        let parsed = Descriptor::<DescriptorPublicKey>::parse_secret(&$descriptors.0).expect("Failed to parse descriptor in `testutils!(@external)`").0;
-        parsed.derive(&[bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()]).address(bitcoin::Network::Regtest).expect("No address form")
+        let secp = Secp256k1::new();
+        let deriv_ctx = DescriptorPublicKeyCtx::new(&secp, bitcoin::util::bip32::ChildNumber::from_normal_idx(0).unwrap());
+
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&$descriptors.0).expect("Failed to parse descriptor in `testutils!(@external)`").0;
+        parsed.derive(bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()).address(bitcoin::Network::Regtest, deriv_ctx).expect("No address form")
     });
     ( @internal $descriptors:expr, $child:expr ) => ({
         use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
 
-        let parsed = Descriptor::<DescriptorPublicKey>::parse_secret(&$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
-        parsed.derive(&[bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()]).address(bitcoin::Network::Regtest).expect("No address form")
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
+        parsed.derive(bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()).address(bitcoin::Network::Regtest).expect("No address form")
     });
     ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
     ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
@@ -268,7 +270,7 @@ impl TestClient {
     pub fn new() -> Self {
         let url = env::var("MAGICAL_RPC_URL").unwrap_or("127.0.0.1:18443".to_string());
         let client = RpcClient::new(format!("http://{}", url), get_auth()).unwrap();
-        let electrum = ElectrumClient::new(&get_electrum_url(), None).unwrap();
+        let electrum = ElectrumClient::new(&get_electrum_url()).unwrap();
 
         TestClient { client, electrum }
     }
@@ -343,7 +345,7 @@ impl TestClient {
             .unwrap();
 
         if let Some(num) = meta_tx.min_confirmations {
-            self.generate(num);
+            self.generate(num, None);
         }
 
         let monitor_script = Address::from_str(&meta_tx.output[0].to_address)
@@ -388,7 +390,7 @@ impl TestClient {
         trace!("getblocktemplate: {:#?}", block_template);
 
         let header = BlockHeader {
-            version: block_template["version"].as_u64().unwrap() as u32,
+            version: block_template["version"].as_i64().unwrap() as i32,
             prev_blockhash: BlockHash::from_hex(
                 block_template["previousblockhash"].as_str().unwrap(),
             )
@@ -466,9 +468,9 @@ impl TestClient {
         block.header.block_hash().to_hex()
     }
 
-    pub fn generate(&mut self, num_blocks: u64) {
-        let our_addr = self.get_new_address(None, None).unwrap();
-        let hashes = self.generate_to_address(num_blocks, &our_addr).unwrap();
+    pub fn generate(&mut self, num_blocks: u64, address: Option<Address>) {
+        let address = address.unwrap_or_else(|| self.get_new_address(None, None).unwrap());
+        let hashes = self.generate_to_address(num_blocks, &address).unwrap();
         let best_hash = hashes.last().unwrap();
         let height = self.get_block_info(best_hash).unwrap().height;
 
@@ -507,7 +509,7 @@ impl TestClient {
 
     pub fn reorg(&mut self, num_blocks: u64) {
         self.invalidate(num_blocks);
-        self.generate(num_blocks);
+        self.generate(num_blocks, None);
     }
 
     pub fn get_node_address(&self, address_type: Option<AddressType>) -> Address {