Bump version to 0.2.1-dev

The syntax now is pretty much the same as the normal descriptor syntax,
with the only difference that modifiers cannot be grouped together (i.e.
`sdv:older(144)` must be turned into `s:d:v:older(144)`.

.github/pull_request_template.md

@@ -0,0 +1,30 @@
+<!-- You can erase any parts of this template not applicable to your Pull Request. -->
+
+### Description
+
+<!-- Describe the purpose of this PR, what's being adding and/or fixed -->
+
+### Notes to the reviewers
+
+<!-- In this section you can include notes directed to the reviewers, like explaining why some parts
+of the PR were done in a specific way -->
+
+### Checklists
+
+#### All Submissions:
+
+* [ ] I've signed all my commits
+* [ ] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
+* [ ] I ran `cargo fmt` and `cargo clippy` before committing
+
+#### New Features:
+
+* [ ] I've added tests for the new feature
+* [ ] I've added docs for the new feature
+* [ ] I've updated `CHANGELOG.md`
+
+#### Bugfixes:
+
+* [ ] This pull request breaks the existing API
+* [ ] I've added tests to reproduce the issue which are now passing
+* [ ] I'm linking the issue being fixed by this PR

.github/workflows/audit.yml

@@ -0,0 +1,19 @@
+name: Audit
+
+on:
+  push:
+    paths:
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+  schedule:
+    - cron: '0 0 * * 0' # Once per week
+
+jobs:
+
+  security_audit:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/code_coverage.yml

@@ -0,0 +1,27 @@
+on: [push]
+
+name: Code Coverage
+
+jobs:
+  tarpaulin-codecov:
+    name: Tarpaulin to codecov.io
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+
+      - name: Install tarpaulin
+        run: cargo install cargo-tarpaulin
+      - name: Tarpaulin
+        run: cargo tarpaulin --features all-keys,cli-utils,compiler,esplora,compact_filters --run-types Tests,Doctests --exclude-files "testutils/*" --out Xml
+
+      - name: Publish to codecov.io
+        uses: codecov/codecov-action@v1.0.15
+        with:
+          fail_ci_if_error: true
+          file: ./cobertura.xml

.github/workflows/cont_integration.yml

@@ -0,0 +1,148 @@
+on: [push, pull_request]
+
+name: CI
+
+jobs:
+
+  build-test:
+    name: Build and test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+          - 1.45.0 # MSRV
+        features:
+          - default
+          - minimal
+          - all-keys
+          - minimal,esplora
+          - key-value-db
+          - electrum
+          - compact_filters
+          - cli-utils,esplora,key-value-db,electrum
+          - compiler
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Generate cache key
+        run: echo "${{ matrix.rust }} ${{ matrix.features }}" | tee .cache_key
+      - name: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('.cache_key') }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Set default toolchain
+        run: rustup default ${{ matrix.rust }}
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add clippy
+        run: rustup component add clippy
+      - name: Build
+        run: cargo build --features ${{ matrix.features }} --no-default-features
+      - name: Clippy
+        run: cargo clippy -- -D warnings
+      - name: Test
+        run: cargo test --features ${{ matrix.features }} --no-default-features
+
+  test-readme-examples:
+    name: Test README.md examples
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-test-md-docs-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Set default toolchain
+        run: rustup default nightly
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Test
+        run: cargo test --features test-md-docs --no-default-features -- doctest::ReadmeDoctests
+
+  test-electrum:
+    name: Test electrum
+    runs-on: ubuntu-16.04
+    container: bitcoindevkit/electrs
+    env:
+      MAGICAL_RPC_AUTH: USER_PASS
+      MAGICAL_RPC_USER: admin
+      MAGICAL_RPC_PASS: passw
+      MAGICAL_RPC_URL: 127.0.0.1:18443
+      MAGICAL_ELECTRUM_URL: tcp://127.0.0.1:60401
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Install rustup
+        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - name: Set default toolchain
+        run: $HOME/.cargo/bin/rustup default stable
+      - name: Set profile
+        run: $HOME/.cargo/bin/rustup set profile minimal
+      - name: Start core
+        run: ./ci/start-core.sh
+      - name: Test
+        run: $HOME/.cargo/bin/cargo test --features test-electrum --no-default-features
+
+  check-wasm:
+    name: Check WASM
+    runs-on: ubuntu-16.04
+    env:
+      CC: clang-10
+      CFLAGS: -I/usr/include
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+        # Install a recent version of clang that supports wasm32
+      - run: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
+      - run: sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" || exit 1
+      - run: sudo apt-get update || exit 1
+      - run: sudo apt-get install -y clang-10 libc6-dev-i386 || exit 1
+      - name: Set default toolchain
+        run: rustup default stable
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add target wasm32
+        run: rustup target add wasm32-unknown-unknown
+      - name: Check
+        run: cargo check --target wasm32-unknown-unknown --features cli-utils,esplora --no-default-features
+
+  fmt:
+    name: Rust fmt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set default toolchain
+        run: rustup default stable
+      - name: Set profile
+        run: rustup set profile minimal
+      - name: Add clippy
+        run: rustup component add rustfmt
+      - name: Check fmt
+        run: cargo fmt --all -- --check

.github/workflows/nightly_docs.yml

@@ -0,0 +1,64 @@
+name: Publish Nightly Docs
+
+on: [push, pull_request]
+
+jobs:
+  build_docs:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Setup cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: nightly-docs-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - name: Install nightly toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+      - name: Build docs
+        uses: actions-rs/cargo@v1
+        with:
+          command: rustdoc
+          args: --verbose --features=compiler,electrum,esplora,compact_filters,key-value-db,all-keys -- --cfg docsrs
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: built-docs
+          path: ./target/doc/*
+
+  publish_docs:
+    name: 'Publish docs'
+    if: github.ref == 'refs/heads/master'
+    needs: [build_docs]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout `bitcoindevkit.org`
+        uses: actions/checkout@v2
+        with:
+          ssh-key: ${{ secrets.DOCS_PUSH_SSH_KEY }}
+          repository: bitcoindevkit/bitcoindevkit.org
+          ref: master
+      - name: Create directories
+        run: mkdir -p ./static/docs-rs/bdk/nightly
+      - name: Remove old latest
+        run: rm -rf ./static/docs-rs/bdk/nightly/latest
+      - name: Download built docs
+        uses: actions/download-artifact@v1
+        with:
+          name: built-docs
+          path: ./static/docs-rs/bdk/nightly/latest
+      - name: Configure git
+        run: git config user.email "github-actions@github.com" && git config user.name "github-actions"
+      - name: Commit
+        continue-on-error: true         # If there's nothing to commit this step fails, but it's fine
+        run: git add ./static && git commit -m "Publish autogenerated nightly docs"
+      - name: Push
+        run: git push origin master

.gitignore

@@ -2,3 +2,4 @@
 Cargo.lock
 
 *.swp
+.idea

.travis.yml

@@ -1,22 +0,0 @@
-language: rust
-rust:
-  - stable
-#  - 1.31.0
-#  - 1.22.0
-before_script:
-  - rustup component add rustfmt
-script:
-  - cargo fmt -- --check --verbose
-  - cargo test --verbose --all
-  - cargo build --verbose --all
-  - cargo build --verbose --no-default-features --features=minimal
-  - cargo build --verbose --no-default-features --features=minimal,esplora
-  - cargo build --verbose --no-default-features --features=key-value-db
-  - cargo build --verbose --no-default-features --features=electrum
-
-notifications:
-  email: false
-
-before_cache:
-  - rm -rf "$TRAVIS_HOME/.cargo/registry/src"
-cache: cargo

CHANGELOG.md

@@ -0,0 +1,213 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Project
+#### Added
+- Add CONTRIBUTING.md
+- Add a Discord badge to the README
+- Add code coverage github actions workflow
+- Add scheduled audit check in CI
+- Add CHANGELOG.md
+
+#### Changed
+- Rename the library to `bdk`
+- Rename `ScriptType` to `KeychainKind`
+- Prettify README examples on github
+- Change CI to github actions
+- Bump rust-bitcoin to 0.25, fix Cargo dependencies
+- Enable clippy for stable and tests by default
+- Switch to "mainline" rust-miniscript
+- Generate a different cache key for every CI job
+- Fix to at least bitcoin ^0.25.2
+
+#### Fixed
+- Fix or ignore clippy warnings for all optional features except compact_filters
+- Pin cc version because last breaks rocksdb build
+
+### Blockchain
+#### Added
+- Add a trait to create `Blockchain`s from a configuration
+- Add an `AnyBlockchain` enum to allow switching at runtime
+- Document `AnyBlockchain` and `ConfigurableBlockchain`
+- Use our Instant struct to be compatible with wasm
+- Make esplora call in parallel
+- Allow to set concurrency in Esplora config and optionally pass it in repl
+
+#### Fixed
+- Fix receiving a coinbase using Electrum/Esplora
+- Use proper type for EsploraHeader, make conversion to BlockHeader infallible
+- Eagerly unwrap height option, save one collect
+
+#### Changed
+- Simplify the architecture of blockchain traits
+- Improve sync
+- Remove unused varaint HeaderParseFail
+
+### CLI
+#### Added
+- Conditionally remove cli args according to enabled feature
+
+#### Changed
+- Add max_addresses param in sync
+- Split the internal and external policy paths
+
+### Database
+#### Added
+- Add `AnyDatabase` and `ConfigurableDatabase` traits
+
+### Descriptor
+#### Added
+- Add a macro to write descriptors from code
+- Add descriptor templates, add `DerivableKey`
+- Add ToWalletDescriptor trait tests
+- Add support for `sortedmulti` in `descriptor!`
+- Add ExtractPolicy trait tests
+- Add get_checksum tests, cleanup tests
+- Add descriptor macro tests
+
+#### Changes
+- Improve the descriptor macro, add traits for key and descriptor types
+
+#### Fixes
+- Fix the recovery of a descriptor given a PSBT
+
+### Keys
+#### Added
+- Add BIP39 support
+- Take `ScriptContext` into account when converting keys
+- Add a way to restrict the networks in which keys are valid
+- Add a trait for keys that can be generated
+- Fix entropy generation
+- Less convoluted entropy generation
+- Re-export tiny-bip39
+- Implement `GeneratableKey` trait for `bitcoin::PrivateKey`
+- Implement `ToDescriptorKey` trait for `GeneratedKey`
+- Add a shortcut to generate keys with the default options
+
+#### Fixed
+- Fix all-keys and cli-utils tests
+
+### Wallet
+#### Added
+- Allow to define static fees for transactions Fixes #137
+- Merging two match expressions for fee calculation
+- Incorporate RBF rules into utxo selection function
+- Add Branch and Bound coin selection
+- Add tests for BranchAndBoundCoinSelection::coin_select
+- Add tests for BranchAndBoundCoinSelection::bnb
+- Add tests for BranchAndBoundCoinSelection::single_random_draw
+- Add test that shwpkh populates witness_utxo
+- Add witness and redeem scripts to PSBT outputs
+- Add an option to include `PSBT_GLOBAL_XPUB`s in PSBTs
+- Eagerly finalize inputs
+
+#### Changed
+- Use collect to avoid iter unwrapping Options
+- Make coin_select take may/must use utxo lists
+- Improve `CoinSelectionAlgorithm`
+- Refactor `Wallet::bump_fee()`
+- Default to SIGHASH_ALL if not specified
+- Replace ChangeSpendPolicy::filter_utxos with a predicate
+- Make 'unspendable' into a HashSet
+- Stop implicitly enforcing manaul selection by .add_utxo
+- Rename DumbCS to LargestFirstCoinSelection
+- Rename must_use_utxos to required_utxos
+- Rename may_use_utxos to optional_uxtos
+- Rename get_must_may_use_utxos to preselect_utxos
+- Remove redundant Box around address validators
+- Remove redundant Box around signers
+- Make Signer and AddressValidator Send and Sync
+- Split `send_all` into `set_single_recipient` and `drain_wallet`
+- Use TXIN_DEFAULT_WEIGHT constant in coin selection
+- Replace `must_use` with `required` in coin selection
+- Take both spending policies into account in create_tx
+- Check last derivation in cache to avoid recomputation
+- Use the branch-and-bound cs by default
+- Make coin_select return UTXOs instead of TxIns
+- Build output lookup inside complete transaction
+- Don't wrap SignersContainer arguments in Arc
+- More consistent references with 'signers' variables
+
+#### Fixed
+- Fix signing for `ShWpkh` inputs
+- Fix the recovery of a descriptor given a PSBT
+
+### Examples
+#### Added
+- Support esplora blockchain source in repl
+
+#### Changed
+- Revert back the REPL example to use Electrum
+- Remove the `magic` alias for `repl`
+- Require esplora feature for repl example
+
+#### Security
+- Use dirs-next instead of dirs since the latter is unmantained
+
+## [0.1.0-beta.1] - 2020-09-08
+
+### Blockchain
+#### Added
+- Lightweight Electrum client with SSL/SOCKS5 support
+- Add a generalized "Blockchain" interface
+- Add Error::OfflineClient
+- Add the Esplora backend
+- Use async I/O in the various blockchain impls
+- Compact Filters blockchain implementation
+- Add support for Tor
+- Impl OnlineBlockchain for types wrapped in Arc
+
+### Database
+#### Added
+- Add a generalized database trait and a Sled-based implementation
+- Add an in-memory database
+
+### Descriptor
+#### Added
+- Wrap Miniscript descriptors to support xpubs
+- Policy and contribution
+- Transform a descriptor into its "public" version
+- Use `miniscript::DescriptorPublicKey`
+
+### Macros
+#### Added
+- Add a feature to enable the async interface on non-wasm32 platforms
+
+### Wallet
+#### Added
+- Wallet logic
+- Add `assume_height_reached` in PSBTSatisfier
+- Add an option to change the assumed current height
+- Specify the policy branch with a map
+- Add a few commands to handle psbts
+- Add hd_keypaths to outputs
+- Add a `TxBuilder` struct to simplify `create_tx()`'s interface
+- Abstract coin selection in a separate trait
+- Refill the address pool whenever necessary
+- Implement the wallet import/export format from FullyNoded
+- Add a type convert fee units, add `Wallet::estimate_fee()`
+- TxOrdering, shuffle/bip69 support
+- Add RBF and custom versions in TxBuilder
+- Allow limiting the use of internal utxos in TxBuilder
+- Add `force_non_witness_utxo()` to TxBuilder
+- RBF and add a few tests
+- Add AddressValidators
+- Add explicit ordering for the signers
+- Support signing the whole tx instead of individual inputs
+- Create a PSBT signer from an ExtendedDescriptor
+
+### Examples
+#### Added
+- Add REPL broadcast command
+- Add a miniscript compiler CLI
+- Expose list_transactions() in the REPL
+- Use `MemoryDatabase` in the compiler example
+- Make the REPL return JSON
+
+[unreleased]: https://github.com/bitcoindevkit/bdk/compare/0.1.0-beta.1...HEAD
+[0.1.0-beta.1]: https://github.com/bitcoindevkit/bdk/compare/96c87ea5...0.1.0-beta.1

CONTRIBUTING.md

@@ -0,0 +1,103 @@
+Contributing to BDK
+==============================
+
+The BDK project operates an open contributor model where anyone is welcome to
+contribute towards development in the form of peer review, documentation,
+testing and patches.
+
+Anyone is invited to contribute without regard to technical experience,
+"expertise", OSS experience, age, or other concern. However, the development of
+cryptocurrencies demands a high-level of rigor, adversarial thinking, thorough
+testing and risk-minimization.
+Any bug may cost users real money. That being said, we deeply welcome people
+contributing for the first time to an open source project or pick up Rust while
+contributing. Don't be shy, you'll learn.
+
+Communications Channels
+-----------------------
+
+Communication about BDK happens primarily on the [BDK Discord](https://discord.gg/dstn4dQ).
+
+Discussion about code base improvements happens in GitHub [issues](https://github.com/bitcoindevkit/bdk/issues) and
+on [pull requests](https://github.com/bitcoindevkit/bdk/pulls).
+
+Contribution Workflow
+---------------------
+
+The codebase is maintained using the "contributor workflow" where everyone
+without exception contributes patch proposals using "pull requests". This
+facilitates social contribution, easy testing and peer review.
+
+To contribute a patch, the worflow is a as follows:
+
+  1. Fork Repository
+  2. Create topic branch
+  3. Commit patches
+
+In general commits should be atomic and diffs should be easy to read.
+For this reason do not mix any formatting fixes or code moves with actual code
+changes. Further, each commit, individually, should compile and pass tests, in
+order to ensure git bisect and other automated tools function properly.
+
+When adding a new feature, thought must be given to the long term technical
+debt.
+Every new feature should be covered by functional tests where possible.
+
+When refactoring, structure your PR to make it easy to review and don't
+hesitate to split it into multiple small, focused PRs.
+
+The Minimal Supported Rust Version is 1.45 (enforced by our CI).
+
+Commits should cover both the issue fixed and the solution's rationale.
+These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind.
+
+To facilitate communication with other contributors, the project is making use
+of GitHub's "assignee" field. First check that no one is assigned and then
+comment suggesting that you're working on it. If someone is already assigned,
+don't hesitate to ask if the assigned party or previous commenters are still
+working on it if it has been awhile.
+
+Peer review
+-----------
+
+Anyone may participate in peer review which is expressed by comments in the
+pull request. Typically reviewers will review the code for obvious errors, as
+well as test out the patch set and opine on the technical merits of the patch.
+PR should be reviewed first on the conceptual level before focusing on code
+style or grammar fixes.
+
+Coding Conventions
+------------------
+
+This codebase uses spaces, not tabs.
+Use `cargo fmt` with the default settings to format code before committing.
+This is also enforced by the CI.
+
+Security
+--------
+
+Security is a high priority of BDK; disclosure of security vulnerabilites helps
+prevent user loss of funds.
+
+Note that BDK is currently considered "pre-production" during this time, there
+is no special handling of security issues. Please simply open an issue on
+Github.
+
+Testing
+-------
+
+Related to the security aspect, BDK developers take testing very seriously.
+Due to the modular nature of the project, writing new functional tests is easy
+and good test coverage of the codebase is an important goal.
+Refactoring the project to enable fine-grained unit testing is also an ongoing
+effort.
+
+Going further
+-------------
+
+You may be interested by Jon Atacks guide on [How to review Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-review-bitcoin-core-prs.md)
+and [How to make Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-make-bitcoin-core-prs.md).
+While there are differences between the projects in terms of context and
+maturity, many of the suggestions offered apply to this project.
+
+Overall, have fun :)

Cargo.toml

@@ -1,63 +1,96 @@
 [package]
-name = "magical-bitcoin-wallet"
-version = "0.1.0"
+name = "bdk"
+version = "0.2.1-dev"
 edition = "2018"
-authors = ["Riccardo Casatta <riccardo@casatta.it>", "Alekos Filini <alekos.filini@gmail.com>"]
+authors = ["Alekos Filini <alekos.filini@gmail.com>", "Riccardo Casatta <riccardo@casatta.it>"]
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk"
+description = "A modern, lightweight, descriptor-based wallet library"
+keywords = ["bitcoin", "wallet", "descriptor", "psbt"]
+readme = "README.md"
+license-file = "LICENSE"
 
 [dependencies]
+bdk-macros = "0.2"
 log = "^0.4"
-bitcoin = { version = "0.23", features = ["use-serde"] }
-miniscript = { version = "0.12" }
+miniscript = "4.0"
+bitcoin = { version = "^0.25.2", features = ["use-serde"] }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
-base64 = "^0.11"
-async-trait = "0.1"
+rand = "^0.7"
 
 # Optional dependencies
-sled = { version = "0.31.0", optional = true }
-electrum-client = { git = "https://github.com/MagicalBitcoin/rust-electrum-client.git", optional = true }
+sled = { version = "0.34", optional = true }
+electrum-client = { version = "0.4.0-beta.1", optional = true }
 reqwest = { version = "0.10", optional = true, features = ["json"] }
 futures = { version = "0.3", optional = true }
 clap = { version = "2.33", optional = true }
+base64 = { version = "^0.11", optional = true }
+async-trait = { version = "0.1", optional = true }
+rocksdb = { version = "0.14", optional = true }
+# pin cc version to 1.0.62 because 1.0.63 break rocksdb build
+cc = { version = "=1.0.62", optional = true }
+socks = { version = "0.3", optional = true }
+lazy_static = { version = "1.4", optional = true }
+tiny-bip39 = { version = "^0.8", optional = true }
+structopt = { version = "^0.3", optional = true }
+
+# Platform-specific dependencies
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+tokio = { version = "0.2", features = ["rt-core"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+async-trait = "0.1"
+js-sys = "0.3"
+rand = { version = "^0.7", features = ["wasm-bindgen"] }
 
 [features]
 minimal = []
-compiler = ["miniscript/compiler"]
+compiler = ["clap", "miniscript/compiler"]
 default = ["key-value-db", "electrum"]
 electrum = ["electrum-client"]
 esplora = ["reqwest", "futures"]
+compact_filters = ["rocksdb", "socks", "lazy_static", "cc"]
 key-value-db = ["sled"]
-cli-utils = ["clap"]
-multiparty = []
+cli-utils = ["clap", "base64", "structopt"]
+async-interface = ["async-trait"]
+all-keys = ["keys-bip39"]
+keys-bip39 = ["tiny-bip39"]
+
+# Debug/Test features
+debug-proc-macros = ["bdk-macros/debug", "bdk-testutils-macros/debug"]
+test-electrum = ["electrum"]
+test-md-docs = ["base64", "electrum"]
 
 [dev-dependencies]
-tokio = { version = "0.2", features = ["macros"] }
+bdk-testutils = "0.2"
+bdk-testutils-macros = "0.2"
+serial_test = "0.4"
 lazy_static = "1.4"
 rustyline = "6.0"
-dirs = "2.0"
+dirs-next = "2.0"
 env_logger = "0.7"
-rand = "0.7"
 
 [[example]]
 name = "repl"
 required-features = ["cli-utils"]
 [[example]]
-name = "psbt"
-[[example]]
 name = "parse_descriptor"
+[[example]]
+name = "address_validator"
 
 [[example]]
 name = "miniscriptc"
 path = "examples/compiler.rs"
 required-features = ["compiler"]
 
-[[example]]
-name = "multiparty"
-required-features = ["multiparty","compiler"]
-
-# Provide a more user-friendly alias for the REPL
-[[example]]
-name = "magic"
-path = "examples/repl.rs"
-required-features = ["cli-utils"]
+[workspace]
+members = ["macros", "testutils", "testutils-macros"]
 
+# Generate docs with nightly to add the "features required" badge
+# https://stackoverflow.com/questions/61417452/how-to-get-a-feature-requirement-tag-in-the-documentation-generated-by-cargo-do
+[package.metadata.docs.rs]
+features = ["compiler", "electrum", "esplora", "compact_filters", "key-value-db", "all-keys"]
+# defines the configuration attribute `docsrs`
+rustdoc-args = ["--cfg", "docsrs"]

DEVELOPMENT_CYCLE.md

@@ -0,0 +1,46 @@
+# Development Cycle
+
+This project follows a regular releasing schedule similar to the one [used by the Rust language](https://doc.rust-lang.org/book/appendix-07-nightly-rust.html). In short, this means that a new release is made at a regular
+cadence, with all the feature/bugfixes that made it to `master` in time. This ensures that we don't keep delaying releases waiting for "just one more little thing".
+
+We decided to maintain a faster release cycle while the library is still in "beta", i.e. before release `1.0.0`: since we are constantly adding new features and, even more importantly, fixing issues, we want developers
+to have access to those updates as fast as possible. For this reason we will make a release **every 4 weeks**.
+
+Once the project will have reached a more mature state (>= `1.0.0`), we will very likely switch to longer release cycles of **6 weeks**.
+
+The "feature freeze" will happen **one week before the release date**. This means a new branch will be created originating from the `master` tip at that time, and in that branch we will stop adding new features and only focus
+on ensuring the ones we've added are working properly.
+
+```
+master:           - - - - * - - - * - - - - - - * - - - * ...
+                          |      /              |       |
+release/0.x.0:            * - - #               |       |
+                                                |      /
+release/0.y.0:                                  * - - #
+```
+
+As soon as the release is tagged and published, the `release` branch will be merged back into `master` to update the version in the `Cargo.toml` to apply the new `Cargo.toml` version and all the other fixes made during the feature
+freeze window.
+
+## Making the Release
+
+What follows are notes and procedures that maintaners can refer to when making releases. All the commits and tags must be signed and, ideally, also [timestamped](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md).
+
+Pre-`v1.0.0` our "major" releases only affect the "minor" semver value. Accordingly, our "minor" releases will only affect the "patch" value.
+
+1. Create a new branch called `release/x.y.z` from `master`. Double check that your local `master` is up-to-date with the upstream repo before doing so.
+2. Make a commit on the release branch to bump the version to `x.y.z-rc.1`. The message should be "Bump version to x.y.z-rc.1".
+3. Push the new branch to `bitcoindevkit/bdk` on GitHub.
+4. During the one week of feature freeze run additional tests on the release branch
+5. If a bug is found:
+    - If it's a minor issue you can just fix it in the release branch, since it will be merged back to `master` eventually
+    - For bigger issues you can fix them on `master` and then *cherry-pick* the commit to the release branch
+6. On release day, make a commit on the release branch to bump the version to `x.y.z`. The message should be "Bump version to x.y.z".
+7. Add a tag to this commit. The tag name should be `vx.y.z` (for example `v0.5.0`), and the message "Release x.y.z". Make sure the tag is signed, for extra safety use the explicit `--sign` flag.
+8. Push the new commits to the upstream release branch, wait for the CI to finish one last time.
+9. Publish **all** the updated crates to crates.io.
+10. Make a new commit to bump the version value to `x.y.(z+1)-dev`. The message should be "Bump version to x.y.(z+1)-dev".
+11. Merge the release branch back into `master`.
+12. Make sure the new release shows up on crates.io and that the docs are built correctly on docs.rs.
+13. Announce the release on Twitter, Discord and Telegram.
+14. Celebrate :tada:

README.md

@@ -1,7 +1,148 @@
-# Magical Bitcoin Wallet
+<div align="center">
+  <h1>BDK</h1>
 
-A modern, lightweight, descriptor-based wallet written in Rust!
+  <img src="./static/bdk.svg" width="220" />
 
-## Getting Started
+  <p>
+    <strong>A modern, lightweight, descriptor-based wallet library written in Rust!</strong>
+  </p>
 
-See the documentation at [magicalbitcoin.org](https://magicalbitcoin.org)
+  <p>
+    <!-- <a href="https://crates.io/crates/magical"><img alt="Crate Info" src="https://img.shields.io/crates/v/magical.svg"/></a> -->
+    <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
+    <a href="https://codecov.io/gh/bitcoindevkit/bdk"><img src="https://codecov.io/gh/bitcoindevkit/bdk/branch/master/graph/badge.svg"/></a>
+    <a href="https://bitcoindevkit.org/docs-rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
+    <a href="https://blog.rust-lang.org/2020/07/16/Rust-1.45.0.html"><img alt="Rustc Version 1.45+" src="https://img.shields.io/badge/rustc-1.45%2B-lightgrey.svg"/></a>
+    <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
+  </p>
+
+  <h4>
+    <a href="https://bitcoindevkit.org">Project Homepage</a>
+    <span> | </span>
+    <a href="https://bitcoindevkit.org/docs-rs/bdk">Documentation</a>
+  </h4>
+</div>
+
+## About
+
+The `bdk` library aims to be the core building block for Bitcoin wallets of any kind.
+
+* It uses [Miniscript](https://github.com/rust-bitcoin/rust-miniscript) to support descriptors with generalized conditions. This exact same library can be used to build
+  single-sig wallets, multisigs, timelocked contracts and more.
+* It supports multiple blockchain backends and databases, allowing developers to choose exactly what's right for their projects.
+* It's built to be cross-platform: the core logic works on desktop, mobile, and even WebAssembly.
+* It's very easy to extend: developers can implement customized logic for blockchain backends, databases, signers, coin selection, and more, without having to fork and modify this library.
+
+## Examples
+
+### Sync the balance of a descriptor
+
+```rust,no_run
+use bdk::Wallet;
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+
+use bdk::electrum_client::Client;
+
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+    let wallet = Wallet::new(
+        "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+        ElectrumBlockchain::from(client)
+    )?;
+
+    wallet.sync(noop_progress(), None)?;
+
+    println!("Descriptor balance: {} SAT", wallet.get_balance()?);
+
+    Ok(())
+}
+```
+
+### Generate a few addresses
+
+```rust
+use bdk::{Wallet, OfflineWallet};
+use bdk::database::MemoryDatabase;
+
+fn main() -> Result<(), bdk::Error> {
+    let wallet: OfflineWallet<_> = Wallet::new_offline(
+        "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+    )?;
+
+    println!("Address #0: {}", wallet.get_new_address()?);
+    println!("Address #1: {}", wallet.get_new_address()?);
+    println!("Address #2: {}", wallet.get_new_address()?);
+
+    Ok(())
+}
+```
+
+### Create a transaction
+
+```rust,no_run
+use bdk::{FeeRate, TxBuilder, Wallet};
+use bdk::database::MemoryDatabase;
+use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+
+use bdk::electrum_client::Client;
+
+use bitcoin::consensus::serialize;
+
+fn main() -> Result<(), bdk::Error> {
+    let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+    let wallet = Wallet::new(
+        "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+        ElectrumBlockchain::from(client)
+    )?;
+
+    wallet.sync(noop_progress(), None)?;
+
+    let send_to = wallet.get_new_address()?;
+    let (psbt, details) = wallet.create_tx(
+        TxBuilder::with_recipients(vec![(send_to.script_pubkey(), 50_000)])
+            .enable_rbf()
+            .do_not_spend_change()
+            .fee_rate(FeeRate::from_sat_per_vb(5.0))
+    )?;
+
+    println!("Transaction details: {:#?}", details);
+    println!("Unsigned PSBT: {}", base64::encode(&serialize(&psbt)));
+
+    Ok(())
+}
+```
+
+### Sign a transaction
+
+```rust,no_run
+use bdk::{Wallet, OfflineWallet};
+use bdk::database::MemoryDatabase;
+
+use bitcoin::consensus::deserialize;
+
+fn main() -> Result<(), bdk::Error> {
+    let wallet: OfflineWallet<_> = Wallet::new_offline(
+        "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)",
+        Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"),
+        bitcoin::Network::Testnet,
+        MemoryDatabase::default(),
+    )?;
+
+    let psbt = "...";
+    let psbt = deserialize(&base64::decode(psbt).unwrap())?;
+
+    let (signed_psbt, finalized) = wallet.sign(psbt, None)?;
+
+    Ok(())
+}
+```

ci/start-core.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+
+echo "Starting bitcoin node."
+/root/bitcoind -regtest -server -daemon -fallbackfee=0.0002 -rpcuser=admin -rpcpassword=passw -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0
+
+echo "Waiting for bitcoin node."
+until /root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw getblockchaininfo; do
+    sleep 1
+done
+
+echo "Generating 150 bitcoin blocks."
+ADDR=$(/root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw getnewaddress)
+/root/bitcoin-cli -regtest -rpcuser=admin -rpcpassword=passw generatetoaddress 150 $ADDR
+
+echo "Starting electrs node."
+nohup /root/electrs --network regtest --jsonrpc-import &
+sleep 5

examples/address_validator.rs

@@ -0,0 +1,72 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+use std::sync::Arc;
+
+use bdk::bitcoin;
+use bdk::database::MemoryDatabase;
+use bdk::descriptor::HDKeyPaths;
+use bdk::wallet::address_validator::{AddressValidator, AddressValidatorError};
+use bdk::KeychainKind;
+use bdk::{OfflineWallet, Wallet};
+
+use bitcoin::hashes::hex::FromHex;
+use bitcoin::util::bip32::Fingerprint;
+use bitcoin::{Network, Script};
+
+struct DummyValidator;
+impl AddressValidator for DummyValidator {
+    fn validate(
+        &self,
+        keychain: KeychainKind,
+        hd_keypaths: &HDKeyPaths,
+        script: &Script,
+    ) -> Result<(), AddressValidatorError> {
+        let (_, path) = hd_keypaths
+            .values()
+            .find(|(fing, _)| fing == &Fingerprint::from_hex("bc123c3e").unwrap())
+            .ok_or(AddressValidatorError::InvalidScript)?;
+
+        println!(
+            "Validating `{:?}` {} address, script: {}",
+            keychain, path, script
+        );
+
+        Ok(())
+    }
+}
+
+fn main() -> Result<(), bdk::Error> {
+    let descriptor = "sh(and_v(v:pk(tpubDDpWvmUrPZrhSPmUzCMBHffvC3HyMAPnWDSAQNBTnj1iZeJa7BZQEttFiP4DS4GCcXQHezdXhn86Hj6LHX5EDstXPWrMaSneRWM8yUf6NFd/*),after(630000)))";
+    let mut wallet: OfflineWallet<_> =
+        Wallet::new_offline(descriptor, None, Network::Regtest, MemoryDatabase::new())?;
+
+    wallet.add_address_validator(Arc::new(DummyValidator));
+
+    wallet.get_new_address()?;
+    wallet.get_new_address()?;
+    wallet.get_new_address()?;
+
+    Ok(())
+}

examples/compiler.rs

@@ -1,27 +1,46 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+extern crate bdk;
 extern crate bitcoin;
 extern crate clap;
 extern crate log;
-extern crate magical_bitcoin_wallet;
 extern crate miniscript;
-extern crate rand;
 extern crate serde_json;
-extern crate sled;
 
 use std::str::FromStr;
 
 use log::info;
 
-use rand::distributions::Alphanumeric;
-use rand::{thread_rng, Rng};
-
 use clap::{App, Arg};
 
 use bitcoin::Network;
 use miniscript::policy::Concrete;
 use miniscript::Descriptor;
 
-use magical_bitcoin_wallet::types::ScriptType;
-use magical_bitcoin_wallet::{OfflineWallet, Wallet};
+use bdk::database::memory::MemoryDatabase;
+use bdk::{KeychainKind, OfflineWallet, Wallet};
 
 fn main() {
     env_logger::init_from_env(
@@ -63,48 +82,32 @@ fn main() {
     info!("Compiling policy: {}", policy_str);
 
     let policy = Concrete::<String>::from_str(&policy_str).unwrap();
-    let compiled = policy.compile().unwrap();
 
     let descriptor = match matches.value_of("TYPE").unwrap() {
-        "sh" => Descriptor::Sh(compiled),
-        "wsh" => Descriptor::Wsh(compiled),
-        "sh-wsh" => Descriptor::ShWsh(compiled),
+        "sh" => Descriptor::Sh(policy.compile().unwrap()),
+        "wsh" => Descriptor::Wsh(policy.compile().unwrap()),
+        "sh-wsh" => Descriptor::ShWsh(policy.compile().unwrap()),
         _ => panic!("Invalid type"),
     };
 
     info!("... Descriptor: {}", descriptor);
 
-    let temp_db = {
-        let mut temp_db = std::env::temp_dir();
-        let rand_string: String = thread_rng().sample_iter(&Alphanumeric).take(15).collect();
-        temp_db.push(rand_string);
+    let database = MemoryDatabase::new();
 
-        let database = sled::open(&temp_db).unwrap();
-
-        let network = match matches.value_of("network") {
-            Some("regtest") => Network::Regtest,
-            Some("testnet") | _ => Network::Testnet,
-        };
-        let wallet: OfflineWallet<_> = Wallet::new_offline(
-            &format!("{}", descriptor),
-            None,
-            network,
-            database.open_tree("").unwrap(),
-        )
-        .unwrap();
-
-        info!("... First address: {}", wallet.get_new_address().unwrap());
-
-        if matches.is_present("parsed_policy") {
-            let spending_policy = wallet.policies(ScriptType::External).unwrap();
-            info!(
-                "... Spending policy:\n{}",
-                serde_json::to_string_pretty(&spending_policy).unwrap()
-            );
-        }
-
-        temp_db
+    let network = match matches.value_of("network") {
+        Some("regtest") => Network::Regtest,
+        Some("testnet") | _ => Network::Testnet,
     };
+    let wallet: OfflineWallet<_> =
+        Wallet::new_offline(&format!("{}", descriptor), None, network, database).unwrap();
 
-    std::fs::remove_dir_all(temp_db).unwrap();
+    info!("... First address: {}", wallet.get_new_address().unwrap());
+
+    if matches.is_present("parsed_policy") {
+        let spending_policy = wallet.policies(KeychainKind::External).unwrap();
+        info!(
+            "... Spending policy:\n{}",
+            serde_json::to_string_pretty(&spending_policy).unwrap()
+        );
+    }
 }

examples/multiparty.rs

@@ -1,96 +0,0 @@
-extern crate bitcoin;
-extern crate clap;
-extern crate log;
-extern crate magical_bitcoin_wallet;
-extern crate miniscript;
-extern crate rand;
-extern crate serde_json;
-extern crate sled;
-
-use std::str::FromStr;
-
-use log::info;
-
-use clap::{App, Arg};
-
-use bitcoin::PublicKey;
-
-use miniscript::policy::Concrete;
-use miniscript::Descriptor;
-
-use magical_bitcoin_wallet::multiparty::{Coordinator, Participant, Peer};
-
-fn main() {
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
-    let matches = App::new("Multiparty Tools")
-        .arg(
-            Arg::with_name("POLICY")
-                .help("Sets the spending policy to compile")
-                .required(true)
-                .index(1),
-        )
-        .arg(
-            Arg::with_name("TYPE")
-                .help("Sets the script type used to embed the compiled policy")
-                .required(true)
-                .index(2)
-                .possible_values(&["sh", "wsh", "sh-wsh"]),
-        )
-        .get_matches();
-
-    let policy_str = matches.value_of("POLICY").unwrap();
-    info!("Compiling policy: {}", policy_str);
-
-    let policy = Concrete::<String>::from_str(&policy_str).unwrap();
-    let compiled = policy.compile().unwrap();
-
-    let descriptor = match matches.value_of("TYPE").unwrap() {
-        "sh" => Descriptor::Sh(compiled),
-        "wsh" => Descriptor::Wsh(compiled),
-        "sh-wsh" => Descriptor::ShWsh(compiled),
-        _ => panic!("Invalid type"),
-    };
-
-    info!("Descriptor: {}", descriptor);
-
-    let mut coordinator: Participant<Coordinator> = Participant::new(descriptor).unwrap();
-    /*let policy = coordinator.policy_for(vec![]).unwrap();
-    info!(
-        "Policy:\n{}",
-        serde_json::to_string_pretty(&policy).unwrap()
-    );*/
-
-    let missing_keys = coordinator.missing_keys();
-    info!("Missing keys: {:?}", missing_keys);
-
-    let pk =
-        PublicKey::from_str("02c65413e56b343a0a31c18d506f1502a17fc64dfbcef6bfb00d1c0d6229bb6f61")
-            .unwrap();
-    coordinator.add_key("Alice", pk.into()).unwrap();
-    coordinator.add_key("Carol", pk.into()).unwrap();
-
-    let for_bob = coordinator.descriptor_for("Bob").unwrap();
-    info!("Descriptor for Bob: {}", for_bob);
-
-    let mut bob_peer: Participant<Peer> = Participant::new(for_bob).unwrap();
-    info!(
-        "Bob's policy: {}",
-        serde_json::to_string(&bob_peer.policy().unwrap().unwrap()).unwrap()
-    );
-    bob_peer.use_key(pk.into()).unwrap();
-    info!("Bob's my_key: {}", bob_peer.my_key().unwrap());
-
-    coordinator.add_key("Bob", pk.into()).unwrap();
-    info!("Coordinator completed: {}", coordinator.completed());
-
-    let coord_map = coordinator.get_map().unwrap();
-
-    let finalized = coordinator.finalize().unwrap();
-    info!("Coordinator final: {}", finalized);
-
-    let bob_finalized = bob_peer.apply_map(coord_map).unwrap();
-    info!("Bob final: {}", bob_finalized);
-}

examples/parse_descriptor.rs

@@ -1,31 +1,60 @@
-extern crate magical_bitcoin_wallet;
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+extern crate bdk;
 extern crate serde_json;
 
-use std::str::FromStr;
+use std::sync::Arc;
 
-use magical_bitcoin_wallet::bitcoin::*;
-use magical_bitcoin_wallet::descriptor::*;
+use bdk::bitcoin::secp256k1::Secp256k1;
+use bdk::bitcoin::util::bip32::ChildNumber;
+use bdk::bitcoin::*;
+use bdk::descriptor::*;
+use bdk::miniscript::DescriptorPublicKeyCtx;
 
 fn main() {
+    let secp = Secp256k1::new();
+
     let desc = "wsh(or_d(\
-                    thresh_m(\
+                    multi(\
                       2,[d34db33f/44'/0'/0']xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*,tprv8ZgxMBicQKsPduL5QnGihpprdHyypMGi4DhimjtzYemu7se5YQNcZfAPLqXRuGHb5ZX2eTQj62oNqMnyxJ7B7wz54Uzswqw8fFqMVdcmVF7/1/*\
                     ),\
                     and_v(vc:pk_h(cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy),older(1000))\
                    ))";
 
-    let extended_desc = ExtendedDescriptor::from_str(desc).unwrap();
+    let (extended_desc, key_map) = ExtendedDescriptor::parse_descriptor(desc).unwrap();
     println!("{:?}", extended_desc);
 
-    let policy = extended_desc.extract_policy().unwrap();
+    let deriv_ctx = DescriptorPublicKeyCtx::new(&secp, ChildNumber::from_normal_idx(42).unwrap());
+
+    let signers = Arc::new(key_map.into());
+    let policy = extended_desc.extract_policy(&signers, &secp).unwrap();
     println!("policy: {}", serde_json::to_string(&policy).unwrap());
 
-    let derived_desc = extended_desc.derive(42).unwrap();
-    println!("{:?}", derived_desc);
-
-    let addr = derived_desc.address(Network::Testnet).unwrap();
+    let addr = extended_desc.address(Network::Testnet, deriv_ctx).unwrap();
     println!("{}", addr);
 
-    let script = derived_desc.witness_script();
+    let script = extended_desc.witness_script(deriv_ctx);
     println!("{:?}", script);
 }

examples/psbt.rs

@@ -1,50 +0,0 @@
-extern crate base64;
-extern crate magical_bitcoin_wallet;
-
-use std::str::FromStr;
-
-use magical_bitcoin_wallet::bitcoin;
-use magical_bitcoin_wallet::descriptor::*;
-use magical_bitcoin_wallet::psbt::*;
-use magical_bitcoin_wallet::signer::Signer;
-
-use bitcoin::consensus::encode::{deserialize, serialize};
-use bitcoin::util::psbt::PartiallySignedTransaction;
-use bitcoin::SigHashType;
-
-fn main() {
-    let desc = "pkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/*)";
-
-    let extended_desc = ExtendedDescriptor::from_str(desc).unwrap();
-
-    let psbt_str = "cHNidP8BAFMCAAAAAd9SiQfxXZ+CKjgjRNonWXsnlA84aLvjxtwCmMfRc0ZbAQAAAAD+////ASjS9QUAAAAAF6kUYJR3oB0lS1M0W1RRMMiENSX45IuHAAAAAAABAPUCAAAAA9I7/OqeFeOFdr5VTLnj3UI/CNRw2eWmMPf7qDv6uIF6AAAAABcWABTG+kgr0g44V0sK9/9FN9oG/CxMK/7///+d0ffphPcV6FE9J/3ZPKWu17YxBnWWTJQyRJs3HUo1gwEAAAAA/v///835mYd9DmnjVnUKd2421MDoZmIxvB4XyJluN3SPUV9hAAAAABcWABRfvwFGp+x/yWdXeNgFs9v0duyeS/7///8CFbH+AAAAAAAXqRSEnTOAjJN/X6ZgR9ftKmwisNSZx4cA4fUFAAAAABl2qRTs6pS4x17MSQ4yNs/1GPsfdlv2NIisAAAAACIGApVE9PPtkcqp8Da43yrXGv4nLOotZdyxwJoTWQxuLxIuCAxfmh4JAAAAAAA=";
-    let psbt_buf = base64::decode(psbt_str).unwrap();
-    let mut psbt: PartiallySignedTransaction = deserialize(&psbt_buf).unwrap();
-
-    let signer = PSBTSigner::from_descriptor(&psbt.global.unsigned_tx, &extended_desc).unwrap();
-
-    for (index, input) in psbt.inputs.iter_mut().enumerate() {
-        for (pubkey, (fing, path)) in &input.hd_keypaths {
-            let sighash = input.sighash_type.unwrap_or(SigHashType::All);
-
-            // Ignore the "witness_utxo" case because we know this psbt is a legacy tx
-            if let Some(non_wit_utxo) = &input.non_witness_utxo {
-                let prev_script = &non_wit_utxo.output
-                    [psbt.global.unsigned_tx.input[index].previous_output.vout as usize]
-                    .script_pubkey;
-                let (signature, sighash) = signer
-                    .sig_legacy_from_fingerprint(index, sighash, fing, path, prev_script)
-                    .unwrap()
-                    .unwrap();
-
-                let mut concat_sig = Vec::new();
-                concat_sig.extend_from_slice(&signature.serialize_der());
-                concat_sig.extend_from_slice(&[sighash as u8]);
-
-                input.partial_sigs.insert(*pubkey, concat_sig);
-            }
-        }
-    }
-
-    println!("signed: {}", base64::encode(&serialize(&psbt)));
-}

examples/repl.rs

@@ -1,27 +1,63 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 use std::fs;
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::sync::Arc;
 
+use bitcoin::Network;
+use clap::AppSettings;
+use log::{debug, info, warn, LevelFilter};
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
+use structopt::StructOpt;
 
-use clap::AppSettings;
+use bdk::bitcoin;
+#[cfg(feature = "esplora")]
+use bdk::blockchain::esplora::EsploraBlockchainConfig;
+use bdk::blockchain::{
+    AnyBlockchain, AnyBlockchainConfig, ConfigurableBlockchain, ElectrumBlockchainConfig,
+};
+use bdk::cli::{self, WalletOpt, WalletSubCommand};
+use bdk::sled;
+use bdk::Wallet;
 
-#[allow(unused_imports)]
-use log::{debug, error, info, trace, LevelFilter};
-
-use bitcoin::Network;
-
-use magical_bitcoin_wallet::bitcoin;
-use magical_bitcoin_wallet::blockchain::ElectrumBlockchain;
-use magical_bitcoin_wallet::cli;
-use magical_bitcoin_wallet::sled;
-use magical_bitcoin_wallet::{Client, Wallet};
+#[derive(Debug, StructOpt, Clone, PartialEq)]
+#[structopt(name = "BDK Wallet", setting = AppSettings::NoBinaryName,
+version = option_env ! ("CARGO_PKG_VERSION").unwrap_or("unknown"),
+author = option_env ! ("CARGO_PKG_AUTHORS").unwrap_or(""))]
+struct ReplOpt {
+    /// Wallet sub-command
+    #[structopt(subcommand)]
+    pub subcommand: WalletSubCommand,
+}
 
 fn prepare_home_dir() -> PathBuf {
     let mut dir = PathBuf::new();
-    dir.push(&dirs::home_dir().unwrap());
-    dir.push(".magical-bitcoin");
+    dir.push(&dirs_next::home_dir().unwrap());
+    dir.push(".bdk-bitcoin");
 
     if !dir.exists() {
         info!("Creating home directory {}", dir.as_path().display());
@@ -32,95 +68,107 @@ fn prepare_home_dir() -> PathBuf {
     dir
 }
 
-#[tokio::main]
-async fn main() {
-    env_logger::init();
+fn main() {
+    let cli_opt: WalletOpt = WalletOpt::from_args();
 
-    let app = cli::make_cli_subcommands();
-    let mut repl_app = app.clone().setting(AppSettings::NoBinaryName);
+    let level = LevelFilter::from_str(cli_opt.log_level.as_str()).unwrap_or(LevelFilter::Info);
+    env_logger::builder().filter_level(level).init();
 
-    let app = cli::add_global_flags(app);
+    let network = Network::from_str(cli_opt.network.as_str()).unwrap_or(Network::Testnet);
+    debug!("network: {:?}", network);
+    if network == Network::Bitcoin {
+        warn!("This is experimental software and not currently recommended for use on Bitcoin mainnet, proceed with caution.")
+    }
 
-    let matches = app.get_matches();
-
-    // TODO
-    // let level = match matches.occurrences_of("v") {
-    //     0 => LevelFilter::Info,
-    //     1 => LevelFilter::Debug,
-    //     _ => LevelFilter::Trace,
-    // };
-
-    let network = match matches.value_of("network") {
-        Some("regtest") => Network::Regtest,
-        Some("testnet") | _ => Network::Testnet,
-    };
-
-    let descriptor = matches.value_of("descriptor").unwrap();
-    let change_descriptor = matches.value_of("change_descriptor");
+    let descriptor = cli_opt.descriptor.as_str();
+    let change_descriptor = cli_opt.change_descriptor.as_deref();
     debug!("descriptors: {:?} {:?}", descriptor, change_descriptor);
 
     let database = sled::open(prepare_home_dir().to_str().unwrap()).unwrap();
-    let tree = database
-        .open_tree(matches.value_of("wallet").unwrap())
-        .unwrap();
+    let tree = database.open_tree(cli_opt.wallet).unwrap();
     debug!("database opened successfully");
 
-    let client = Client::new(matches.value_of("server").unwrap())
-        .await
-        .unwrap();
+    // Try to use Esplora config if "esplora" feature is enabled
+    #[cfg(feature = "esplora")]
+    let config_esplora: Option<AnyBlockchainConfig> = {
+        let esplora_concurrency = cli_opt.esplora_concurrency;
+        cli_opt.esplora.map(|base_url| {
+            AnyBlockchainConfig::Esplora(EsploraBlockchainConfig {
+                base_url: base_url.to_string(),
+                concurrency: Some(esplora_concurrency),
+            })
+        })
+    };
+    #[cfg(not(feature = "esplora"))]
+    let config_esplora = None;
+
+    // Fall back to Electrum config if Esplora config isn't provided
+    let config =
+        config_esplora.unwrap_or(AnyBlockchainConfig::Electrum(ElectrumBlockchainConfig {
+            url: cli_opt.electrum,
+            socks5: cli_opt.proxy,
+            retry: 10,
+            timeout: 10,
+        }));
+
     let wallet = Wallet::new(
         descriptor,
         change_descriptor,
         network,
         tree,
-        ElectrumBlockchain::from(client),
+        AnyBlockchain::from_config(&config).unwrap(),
     )
-    .await
     .unwrap();
+
     let wallet = Arc::new(wallet);
 
-    if let Some(_sub_matches) = matches.subcommand_matches("repl") {
-        let mut rl = Editor::<()>::new();
+    match cli_opt.subcommand {
+        WalletSubCommand::Other(external) if external.contains(&"repl".to_string()) => {
+            let mut rl = Editor::<()>::new();
 
-        // if rl.load_history("history.txt").is_err() {
-        //     println!("No previous history.");
-        // }
+            // if rl.load_history("history.txt").is_err() {
+            //     println!("No previous history.");
+            // }
 
-        loop {
-            let readline = rl.readline(">> ");
-            match readline {
-                Ok(line) => {
-                    if line.trim() == "" {
-                        continue;
+            loop {
+                let readline = rl.readline(">> ");
+                match readline {
+                    Ok(line) => {
+                        if line.trim() == "" {
+                            continue;
+                        }
+                        rl.add_history_entry(line.as_str());
+                        let split_line: Vec<&str> = line.split(" ").collect();
+                        let repl_subcommand: Result<ReplOpt, clap::Error> =
+                            ReplOpt::from_iter_safe(split_line);
+                        debug!("repl_subcommand = {:?}", repl_subcommand);
+
+                        if let Err(err) = repl_subcommand {
+                            println!("{}", err.message);
+                            continue;
+                        }
+
+                        let result = cli::handle_wallet_subcommand(
+                            &Arc::clone(&wallet),
+                            repl_subcommand.unwrap().subcommand,
+                        )
+                        .unwrap();
+                        println!("{}", serde_json::to_string_pretty(&result).unwrap());
                     }
-
-                    rl.add_history_entry(line.as_str());
-                    let matches = repl_app.get_matches_from_safe_borrow(line.split(" "));
-                    if let Err(err) = matches {
-                        println!("{}", err.message);
-                        continue;
+                    Err(ReadlineError::Interrupted) => continue,
+                    Err(ReadlineError::Eof) => break,
+                    Err(err) => {
+                        println!("{:?}", err);
+                        break;
                     }
-
-                    if let Some(s) = cli::handle_matches(&Arc::clone(&wallet), matches.unwrap())
-                        .await
-                        .unwrap()
-                    {
-                        println!("{}", s);
-                    }
-                }
-                Err(ReadlineError::Interrupted) => continue,
-                Err(ReadlineError::Eof) => break,
-                Err(err) => {
-                    println!("{:?}", err);
-                    break;
                 }
             }
-        }
 
-    // rl.save_history("history.txt").unwrap();
-    } else {
-        if let Some(s) = cli::handle_matches(&wallet, matches).await.unwrap() {
-            println!("{}", s);
+            // rl.save_history("history.txt").unwrap();
+        }
+        _ => {
+            let result = cli::handle_wallet_subcommand(&wallet, cli_opt.subcommand).unwrap();
+            println!("{}", serde_json::to_string_pretty(&result).unwrap());
         }
     }
 }

macros/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "bdk-macros"
+version = "0.2.0"
+authors = ["Alekos Filini <alekos.filini@gmail.com>"]
+edition = "2018"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk-macros"
+description = "Supporting macros for `bdk`"
+keywords = ["bdk"]
+license-file = "../LICENSE"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+syn = { version = "1.0", features = ["parsing", "full"] }
+proc-macro2 = "1.0"
+quote = "1.0"
+
+[features]
+debug = ["syn/extra-traits"]
+
+[lib]
+proc-macro = true

macros/src/lib.rs

@@ -0,0 +1,159 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#[macro_use]
+extern crate quote;
+
+use proc_macro::TokenStream;
+
+use syn::spanned::Spanned;
+use syn::{parse, ImplItemMethod, ItemImpl, ItemTrait, Token};
+
+fn add_async_trait(mut parsed: ItemTrait) -> TokenStream {
+    let output = quote! {
+        #[cfg(all(not(target_arch = "wasm32"), not(feature = "async-interface")))]
+        #parsed
+    };
+
+    for mut item in &mut parsed.items {
+        if let syn::TraitItem::Method(m) = &mut item {
+            m.sig.asyncness = Some(Token![async](m.span()));
+        }
+    }
+
+    let output = quote! {
+        #output
+
+        #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
+        #[async_trait(?Send)]
+        #parsed
+    };
+
+    output.into()
+}
+
+fn add_async_method(mut parsed: ImplItemMethod) -> TokenStream {
+    let output = quote! {
+        #[cfg(all(not(target_arch = "wasm32"), not(feature = "async-interface")))]
+        #parsed
+    };
+
+    parsed.sig.asyncness = Some(Token![async](parsed.span()));
+
+    let output = quote! {
+        #output
+
+        #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
+        #parsed
+    };
+
+    output.into()
+}
+
+fn add_async_impl_trait(mut parsed: ItemImpl) -> TokenStream {
+    let output = quote! {
+        #[cfg(all(not(target_arch = "wasm32"), not(feature = "async-interface")))]
+        #parsed
+    };
+
+    for mut item in &mut parsed.items {
+        if let syn::ImplItem::Method(m) = &mut item {
+            m.sig.asyncness = Some(Token![async](m.span()));
+        }
+    }
+
+    let output = quote! {
+        #output
+
+        #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
+        #[async_trait(?Send)]
+        #parsed
+    };
+
+    output.into()
+}
+
+/// Makes a method or every method of a trait "async" only if the target_arch is "wasm32"
+///
+/// Requires the `async-trait` crate as a dependency whenever this attribute is used on a trait
+/// definition or trait implementation.
+#[proc_macro_attribute]
+pub fn maybe_async(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    if let Ok(parsed) = parse(item.clone()) {
+        add_async_trait(parsed)
+    } else if let Ok(parsed) = parse(item.clone()) {
+        add_async_method(parsed)
+    } else if let Ok(parsed) = parse(item) {
+        add_async_impl_trait(parsed)
+    } else {
+        (quote! {
+            compile_error!("#[maybe_async] can only be used on methods, trait or trait impl blocks")
+        })
+        .into()
+    }
+}
+
+/// Awaits if target_arch is "wasm32", does nothing otherwise
+#[proc_macro]
+pub fn maybe_await(expr: TokenStream) -> TokenStream {
+    let expr: proc_macro2::TokenStream = expr.into();
+    let quoted = quote! {
+        {
+            #[cfg(all(not(target_arch = "wasm32"), not(feature = "async-interface")))]
+            {
+                #expr
+            }
+
+            #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
+            {
+                #expr.await
+            }
+        }
+    };
+
+    quoted.into()
+}
+
+/// Awaits if target_arch is "wasm32", uses `tokio::Runtime::block_on()` otherwise
+///
+/// Requires the `tokio` crate as a dependecy with `rt-core` or `rt-threaded` to build on non-wasm32 platforms.
+#[proc_macro]
+pub fn await_or_block(expr: TokenStream) -> TokenStream {
+    let expr: proc_macro2::TokenStream = expr.into();
+    let quoted = quote! {
+        {
+            #[cfg(all(not(target_arch = "wasm32"), not(feature = "async-interface")))]
+            {
+                tokio::runtime::Runtime::new().unwrap().block_on(#expr)
+            }
+
+            #[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
+            {
+                #expr.await
+            }
+        }
+    };
+
+    quoted.into()
+}

src/blockchain/any.rs

@@ -0,0 +1,232 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Runtime-checked blockchain types
+//!
+//! This module provides the implementation of [`AnyBlockchain`] which allows switching the
+//! inner [`Blockchain`] type at runtime.
+//!
+//! ## Example
+//!
+//! In this example both `wallet_electrum` and `wallet_esplora` have the same type of
+//! `Wallet<AnyBlockchain, MemoryDatabase>`. This means that they could both, for instance, be
+//! assigned to a struct member.
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::blockchain::*;
+//! # use bdk::database::MemoryDatabase;
+//! # use bdk::Wallet;
+//! # #[cfg(feature = "electrum")]
+//! # {
+//! let electrum_blockchain = ElectrumBlockchain::from(electrum_client::Client::new("...")?);
+//! let wallet_electrum: Wallet<AnyBlockchain, _> = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     electrum_blockchain.into(),
+//! )?;
+//! # }
+//!
+//! # #[cfg(feature = "esplora")]
+//! # {
+//! let esplora_blockchain = EsploraBlockchain::new("...", None);
+//! let wallet_esplora: Wallet<AnyBlockchain, _> = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     esplora_blockchain.into(),
+//! )?;
+//! # }
+//!
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! When paired with the use of [`ConfigurableBlockchain`], it allows creating wallets with any
+//! blockchain type supported using a single line of code:
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::blockchain::*;
+//! # use bdk::database::MemoryDatabase;
+//! # use bdk::Wallet;
+//! let config = serde_json::from_str("...")?;
+//! let blockchain = AnyBlockchain::from_config(&config)?;
+//! let wallet = Wallet::new(
+//!     "...",
+//!     None,
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//!     blockchain,
+//! )?;
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use super::*;
+
+macro_rules! impl_from {
+    ( $from:ty, $to:ty, $variant:ident, $( $cfg:tt )* ) => {
+        $( $cfg )*
+        impl From<$from> for $to {
+            fn from(inner: $from) -> Self {
+                <$to>::$variant(inner)
+            }
+        }
+    };
+}
+
+macro_rules! impl_inner_method {
+    ( $self:expr, $name:ident $(, $args:expr)* ) => {
+        match $self {
+            #[cfg(feature = "electrum")]
+            AnyBlockchain::Electrum(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "esplora")]
+            AnyBlockchain::Esplora(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "compact_filters")]
+            AnyBlockchain::CompactFilters(inner) => inner.$name( $($args, )* ),
+        }
+    }
+}
+
+/// Type that can contain any of the [`Blockchain`] types defined by the library
+///
+/// It allows switching backend at runtime
+///
+/// See [this module](crate::blockchain::any)'s documentation for a usage example.
+pub enum AnyBlockchain {
+    #[cfg(feature = "electrum")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
+    /// Electrum client
+    Electrum(electrum::ElectrumBlockchain),
+    #[cfg(feature = "esplora")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
+    /// Esplora client
+    Esplora(esplora::EsploraBlockchain),
+    #[cfg(feature = "compact_filters")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
+    /// Compact filters client
+    CompactFilters(compact_filters::CompactFiltersBlockchain),
+}
+
+#[maybe_async]
+impl Blockchain for AnyBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        maybe_await!(impl_inner_method!(self, get_capabilities))
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(
+            self,
+            setup,
+            stop_gap,
+            database,
+            progress_update
+        ))
+    }
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(
+            self,
+            sync,
+            stop_gap,
+            database,
+            progress_update
+        ))
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        maybe_await!(impl_inner_method!(self, get_tx, txid))
+    }
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        maybe_await!(impl_inner_method!(self, broadcast, tx))
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        maybe_await!(impl_inner_method!(self, get_height))
+    }
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        maybe_await!(impl_inner_method!(self, estimate_fee, target))
+    }
+}
+
+impl_from!(electrum::ElectrumBlockchain, AnyBlockchain, Electrum, #[cfg(feature = "electrum")]);
+impl_from!(esplora::EsploraBlockchain, AnyBlockchain, Esplora, #[cfg(feature = "esplora")]);
+impl_from!(compact_filters::CompactFiltersBlockchain, AnyBlockchain, CompactFilters, #[cfg(feature = "compact_filters")]);
+
+/// Type that can contain any of the blockchain configurations defined by the library
+///
+/// This allows storing a single configuration that can be loaded into an [`AnyBlockchain`]
+/// instance. Wallets that plan to offer users the ability to switch blockchain backend at runtime
+/// will find this particularly useful.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub enum AnyBlockchainConfig {
+    #[cfg(feature = "electrum")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
+    /// Electrum client
+    Electrum(electrum::ElectrumBlockchainConfig),
+    #[cfg(feature = "esplora")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
+    /// Esplora client
+    Esplora(esplora::EsploraBlockchainConfig),
+    #[cfg(feature = "compact_filters")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
+    /// Compact filters client
+    CompactFilters(compact_filters::CompactFiltersBlockchainConfig),
+}
+
+impl ConfigurableBlockchain for AnyBlockchain {
+    type Config = AnyBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(match config {
+            #[cfg(feature = "electrum")]
+            AnyBlockchainConfig::Electrum(inner) => {
+                AnyBlockchain::Electrum(electrum::ElectrumBlockchain::from_config(inner)?)
+            }
+            #[cfg(feature = "esplora")]
+            AnyBlockchainConfig::Esplora(inner) => {
+                AnyBlockchain::Esplora(esplora::EsploraBlockchain::from_config(inner)?)
+            }
+            #[cfg(feature = "compact_filters")]
+            AnyBlockchainConfig::CompactFilters(inner) => AnyBlockchain::CompactFilters(
+                compact_filters::CompactFiltersBlockchain::from_config(inner)?,
+            ),
+        })
+    }
+}
+
+impl_from!(electrum::ElectrumBlockchainConfig, AnyBlockchainConfig, Electrum, #[cfg(feature = "electrum")]);
+impl_from!(esplora::EsploraBlockchainConfig, AnyBlockchainConfig, Esplora, #[cfg(feature = "esplora")]);
+impl_from!(compact_filters::CompactFiltersBlockchainConfig, AnyBlockchainConfig, CompactFilters, #[cfg(feature = "compact_filters")]);

src/blockchain/compact_filters/mod.rs

@@ -0,0 +1,580 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Compact Filters
+//!
+//! This module contains a multithreaded implementation of an [`Blockchain`] backend that
+//! uses BIP157 (aka "Neutrino") to populate the wallet's [database](crate::database::Database)
+//! by downloading compact filters from the P2P network.
+//!
+//! Since there are currently very few peers "in the wild" that advertise the required service
+//! flag, this implementation requires that one or more known peers are provided by the user.
+//! No dns or other kinds of peer discovery are done internally.
+//!
+//! Moreover, this module doesn't currently support detecting and resolving conflicts between
+//! messages received by different peers. Thus, it's recommended to use this module by only
+//! connecting to a single peer at a time, optionally by opening multiple connections if it's
+//! desirable to use multiple threads at once to sync in parallel.
+//!
+//! This is an **EXPERIMENTAL** feature, API and other major changes are expected.
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use std::sync::Arc;
+//! # use bitcoin::*;
+//! # use bdk::*;
+//! # use bdk::blockchain::compact_filters::*;
+//! let num_threads = 4;
+//!
+//! let mempool = Arc::new(Mempool::default());
+//! let peers = (0..num_threads)
+//!     .map(|_| {
+//!         Peer::connect(
+//!             "btcd-mainnet.lightning.computer:8333",
+//!             Arc::clone(&mempool),
+//!             Network::Bitcoin,
+//!         )
+//!     })
+//!     .collect::<Result<_, _>>()?;
+//! let blockchain = CompactFiltersBlockchain::new(peers, "./wallet-filters", Some(500_000))?;
+//! # Ok::<(), CompactFiltersError>(())
+//! ```
+
+use std::collections::HashSet;
+use std::fmt;
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+
+#[allow(unused_imports)]
+use log::{debug, error, info, trace};
+
+use bitcoin::network::message_blockdata::Inventory;
+use bitcoin::{Network, OutPoint, Transaction, Txid};
+
+use rocksdb::{Options, SliceTransform, DB};
+
+mod peer;
+mod store;
+mod sync;
+
+use super::{Blockchain, Capability, ConfigurableBlockchain, Progress};
+use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
+use crate::error::Error;
+use crate::types::{KeychainKind, TransactionDetails, UTXO};
+use crate::FeeRate;
+
+use peer::*;
+use store::*;
+use sync::*;
+
+pub use peer::{Mempool, Peer};
+
+const SYNC_HEADERS_COST: f32 = 1.0;
+const SYNC_FILTERS_COST: f32 = 11.6 * 1_000.0;
+const PROCESS_BLOCKS_COST: f32 = 20_000.0;
+
+/// Structure implementing the required blockchain traits
+///
+/// ## Example
+/// See the [`blockchain::compact_filters`](crate::blockchain::compact_filters) module for a usage example.
+#[derive(Debug)]
+pub struct CompactFiltersBlockchain {
+    peers: Vec<Arc<Peer>>,
+    headers: Arc<ChainStore<Full>>,
+    skip_blocks: Option<usize>,
+}
+
+impl CompactFiltersBlockchain {
+    /// Construct a new instance given a list of peers, a path to store headers and block
+    /// filters downloaded during the sync and optionally a number of blocks to ignore starting
+    /// from the genesis while scanning for the wallet's outputs.
+    ///
+    /// For each [`Peer`] specified a new thread will be spawned to download and verify the filters
+    /// in parallel. It's currently recommended to only connect to a single peer to avoid
+    /// inconsistencies in the data returned, optionally with multiple connections in parallel to
+    /// speed-up the sync process.
+    pub fn new<P: AsRef<Path>>(
+        peers: Vec<Peer>,
+        storage_dir: P,
+        skip_blocks: Option<usize>,
+    ) -> Result<Self, CompactFiltersError> {
+        if peers.is_empty() {
+            return Err(CompactFiltersError::NoPeers);
+        }
+
+        let mut opts = Options::default();
+        opts.create_if_missing(true);
+        opts.set_prefix_extractor(SliceTransform::create_fixed_prefix(16));
+
+        let network = peers[0].get_network();
+
+        let cfs = DB::list_cf(&opts, &storage_dir).unwrap_or(vec!["default".to_string()]);
+        let db = DB::open_cf(&opts, &storage_dir, &cfs)?;
+        let headers = Arc::new(ChainStore::new(db, network)?);
+
+        // try to recover partial snapshots
+        for cf_name in &cfs {
+            if !cf_name.starts_with("_headers:") {
+                continue;
+            }
+
+            info!("Trying to recover: {:?}", cf_name);
+            headers.recover_snapshot(cf_name)?;
+        }
+
+        Ok(CompactFiltersBlockchain {
+            peers: peers.into_iter().map(Arc::new).collect(),
+            headers,
+            skip_blocks,
+        })
+    }
+
+    /// Process a transaction by looking for inputs that spend from a UTXO in the database or
+    /// outputs that send funds to a know script_pubkey.
+    fn process_tx<D: BatchDatabase>(
+        &self,
+        database: &mut D,
+        tx: &Transaction,
+        height: Option<u32>,
+        timestamp: u64,
+        internal_max_deriv: &mut Option<u32>,
+        external_max_deriv: &mut Option<u32>,
+    ) -> Result<(), Error> {
+        let mut updates = database.begin_batch();
+
+        let mut incoming: u64 = 0;
+        let mut outgoing: u64 = 0;
+
+        let mut inputs_sum: u64 = 0;
+        let mut outputs_sum: u64 = 0;
+
+        // look for our own inputs
+        for (i, input) in tx.input.iter().enumerate() {
+            if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
+                inputs_sum += previous_output.value;
+
+                if database.is_mine(&previous_output.script_pubkey)? {
+                    outgoing += previous_output.value;
+
+                    debug!("{} input #{} is mine, removing from utxo", tx.txid(), i);
+                    updates.del_utxo(&input.previous_output)?;
+                }
+            }
+        }
+
+        for (i, output) in tx.output.iter().enumerate() {
+            // to compute the fees later
+            outputs_sum += output.value;
+
+            // this output is ours, we have a path to derive it
+            if let Some((keychain, child)) =
+                database.get_path_from_script_pubkey(&output.script_pubkey)?
+            {
+                debug!("{} output #{} is mine, adding utxo", tx.txid(), i);
+                updates.set_utxo(&UTXO {
+                    outpoint: OutPoint::new(tx.txid(), i as u32),
+                    txout: output.clone(),
+                    keychain,
+                })?;
+                incoming += output.value;
+
+                if keychain == KeychainKind::Internal
+                    && (internal_max_deriv.is_none() || child > internal_max_deriv.unwrap_or(0))
+                {
+                    *internal_max_deriv = Some(child);
+                } else if keychain == KeychainKind::External
+                    && (external_max_deriv.is_none() || child > external_max_deriv.unwrap_or(0))
+                {
+                    *external_max_deriv = Some(child);
+                }
+            }
+        }
+
+        if incoming > 0 || outgoing > 0 {
+            let tx = TransactionDetails {
+                txid: tx.txid(),
+                transaction: Some(tx.clone()),
+                received: incoming,
+                sent: outgoing,
+                height,
+                timestamp,
+                fees: inputs_sum.checked_sub(outputs_sum).unwrap_or(0),
+            };
+
+            info!("Saving tx {}", tx.txid);
+            updates.set_tx(&tx)?;
+        }
+
+        database.commit_batch(updates)?;
+
+        Ok(())
+    }
+}
+
+impl Blockchain for CompactFiltersBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        vec![Capability::FullHistory].into_iter().collect()
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        _stop_gap: Option<usize>, // TODO: move to electrum and esplora only
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        let first_peer = &self.peers[0];
+
+        let skip_blocks = self.skip_blocks.unwrap_or(0);
+
+        let cf_sync = Arc::new(CFSync::new(Arc::clone(&self.headers), skip_blocks, 0x00)?);
+
+        let initial_height = self.headers.get_height()?;
+        let total_bundles = (first_peer.get_version().start_height as usize)
+            .checked_sub(skip_blocks)
+            .map(|x| x / 1000)
+            .unwrap_or(0)
+            + 1;
+        let expected_bundles_to_sync = total_bundles
+            .checked_sub(cf_sync.pruned_bundles()?)
+            .unwrap_or(0);
+
+        let headers_cost = (first_peer.get_version().start_height as usize)
+            .checked_sub(initial_height)
+            .unwrap_or(0) as f32
+            * SYNC_HEADERS_COST;
+        let filters_cost = expected_bundles_to_sync as f32 * SYNC_FILTERS_COST;
+
+        let total_cost = headers_cost + filters_cost + PROCESS_BLOCKS_COST;
+
+        if let Some(snapshot) = sync::sync_headers(
+            Arc::clone(&first_peer),
+            Arc::clone(&self.headers),
+            |new_height| {
+                let local_headers_cost =
+                    new_height.checked_sub(initial_height).unwrap_or(0) as f32 * SYNC_HEADERS_COST;
+                progress_update.update(
+                    local_headers_cost / total_cost * 100.0,
+                    Some(format!("Synced headers to {}", new_height)),
+                )
+            },
+        )? {
+            if snapshot.work()? > self.headers.work()? {
+                info!("Applying snapshot with work: {}", snapshot.work()?);
+                self.headers.apply_snapshot(snapshot)?;
+            }
+        }
+
+        let synced_height = self.headers.get_height()?;
+        let buried_height = synced_height
+            .checked_sub(sync::BURIED_CONFIRMATIONS)
+            .unwrap_or(0);
+        info!("Synced headers to height: {}", synced_height);
+
+        cf_sync.prepare_sync(Arc::clone(&first_peer))?;
+
+        let all_scripts = Arc::new(
+            database
+                .iter_script_pubkeys(None)?
+                .into_iter()
+                .map(|s| s.to_bytes())
+                .collect::<Vec<_>>(),
+        );
+
+        let last_synced_block = Arc::new(Mutex::new(synced_height));
+        let synced_bundles = Arc::new(AtomicUsize::new(0));
+        let progress_update = Arc::new(Mutex::new(progress_update));
+
+        let mut threads = Vec::with_capacity(self.peers.len());
+        for peer in &self.peers {
+            let cf_sync = Arc::clone(&cf_sync);
+            let peer = Arc::clone(&peer);
+            let headers = Arc::clone(&self.headers);
+            let all_scripts = Arc::clone(&all_scripts);
+            let last_synced_block = Arc::clone(&last_synced_block);
+            let progress_update = Arc::clone(&progress_update);
+            let synced_bundles = Arc::clone(&synced_bundles);
+
+            let thread = std::thread::spawn(move || {
+                cf_sync.capture_thread_for_sync(
+                    peer,
+                    |block_hash, filter| {
+                        if !filter
+                            .match_any(block_hash, &mut all_scripts.iter().map(AsRef::as_ref))?
+                        {
+                            return Ok(false);
+                        }
+
+                        let block_height = headers.get_height_for(block_hash)?.unwrap_or(0);
+                        let saved_correct_block = match headers.get_full_block(block_height)? {
+                            Some(block) if &block.block_hash() == block_hash => true,
+                            _ => false,
+                        };
+
+                        if saved_correct_block {
+                            Ok(false)
+                        } else {
+                            let mut last_synced_block = last_synced_block.lock().unwrap();
+
+                            // If we download a block older than `last_synced_block`, we update it so that
+                            // we know to delete and re-process all txs starting from that height
+                            if block_height < *last_synced_block {
+                                *last_synced_block = block_height;
+                            }
+
+                            Ok(true)
+                        }
+                    },
+                    |index| {
+                        let synced_bundles = synced_bundles.fetch_add(1, Ordering::SeqCst);
+                        let local_filters_cost = synced_bundles as f32 * SYNC_FILTERS_COST;
+                        progress_update.lock().unwrap().update(
+                            (headers_cost + local_filters_cost) / total_cost * 100.0,
+                            Some(format!(
+                                "Synced filters {} - {}",
+                                index * 1000 + 1,
+                                (index + 1) * 1000
+                            )),
+                        )
+                    },
+                )
+            });
+
+            threads.push(thread);
+        }
+
+        for t in threads {
+            t.join().unwrap()?;
+        }
+
+        progress_update.lock().unwrap().update(
+            (headers_cost + filters_cost) / total_cost * 100.0,
+            Some("Processing downloaded blocks and mempool".into()),
+        )?;
+
+        // delete all txs newer than last_synced_block
+        let last_synced_block = *last_synced_block.lock().unwrap();
+        log::debug!(
+            "Dropping transactions newer than `last_synced_block` = {}",
+            last_synced_block
+        );
+        let mut updates = database.begin_batch();
+        for details in database.iter_txs(false)? {
+            match details.height {
+                Some(height) if (height as usize) < last_synced_block => continue,
+                _ => updates.del_tx(&details.txid, false)?,
+            };
+        }
+        database.commit_batch(updates)?;
+
+        first_peer.ask_for_mempool()?;
+
+        let mut internal_max_deriv = None;
+        let mut external_max_deriv = None;
+
+        for (height, block) in self.headers.iter_full_blocks()? {
+            for tx in &block.txdata {
+                self.process_tx(
+                    database,
+                    tx,
+                    Some(height as u32),
+                    0,
+                    &mut internal_max_deriv,
+                    &mut external_max_deriv,
+                )?;
+            }
+        }
+        for tx in first_peer.get_mempool().iter_txs().iter() {
+            self.process_tx(
+                database,
+                tx,
+                None,
+                0,
+                &mut internal_max_deriv,
+                &mut external_max_deriv,
+            )?;
+        }
+
+        let current_ext = database
+            .get_last_index(KeychainKind::External)?
+            .unwrap_or(0);
+        let first_ext_new = external_max_deriv.map(|x| x + 1).unwrap_or(0);
+        if first_ext_new > current_ext {
+            info!("Setting external index to {}", first_ext_new);
+            database.set_last_index(KeychainKind::External, first_ext_new)?;
+        }
+
+        let current_int = database
+            .get_last_index(KeychainKind::Internal)?
+            .unwrap_or(0);
+        let first_int_new = internal_max_deriv.map(|x| x + 1).unwrap_or(0);
+        if first_int_new > current_int {
+            info!("Setting internal index to {}", first_int_new);
+            database.set_last_index(KeychainKind::Internal, first_int_new)?;
+        }
+
+        info!("Dropping blocks until {}", buried_height);
+        self.headers.delete_blocks_until(buried_height)?;
+
+        progress_update
+            .lock()
+            .unwrap()
+            .update(100.0, Some("Done".into()))?;
+
+        Ok(())
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(self.peers[0]
+            .get_mempool()
+            .get_tx(&Inventory::Transaction(*txid)))
+    }
+
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        self.peers[0].broadcast_tx(tx.clone())?;
+
+        Ok(())
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(self.headers.get_height()? as u32)
+    }
+
+    fn estimate_fee(&self, _target: usize) -> Result<FeeRate, Error> {
+        // TODO
+        Ok(FeeRate::default())
+    }
+}
+
+/// Data to connect to a Bitcoin P2P peer
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct BitcoinPeerConfig {
+    /// Peer address such as 127.0.0.1:18333
+    pub address: String,
+    /// Optional socks5 proxy
+    pub socks5: Option<String>,
+    /// Optional socks5 proxy credentials
+    pub socks5_credentials: Option<(String, String)>,
+}
+
+/// Configuration for a [`CompactFiltersBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct CompactFiltersBlockchainConfig {
+    /// List of peers to try to connect to for asking headers and filters
+    pub peers: Vec<BitcoinPeerConfig>,
+    /// Network used
+    pub network: Network,
+    /// Storage dir to save partially downloaded headers and full blocks
+    pub storage_dir: String,
+    /// Optionally skip initial `skip_blocks` blocks (default: 0)
+    pub skip_blocks: Option<usize>,
+}
+
+impl ConfigurableBlockchain for CompactFiltersBlockchain {
+    type Config = CompactFiltersBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let mempool = Arc::new(Mempool::default());
+        let peers = config
+            .peers
+            .iter()
+            .map(|peer_conf| match &peer_conf.socks5 {
+                None => Peer::connect(&peer_conf.address, Arc::clone(&mempool), config.network),
+                Some(proxy) => Peer::connect_proxy(
+                    peer_conf.address.as_str(),
+                    proxy,
+                    peer_conf
+                        .socks5_credentials
+                        .as_ref()
+                        .map(|(a, b)| (a.as_str(), b.as_str())),
+                    Arc::clone(&mempool),
+                    config.network,
+                ),
+            })
+            .collect::<Result<_, _>>()?;
+
+        Ok(CompactFiltersBlockchain::new(
+            peers,
+            &config.storage_dir,
+            config.skip_blocks,
+        )?)
+    }
+}
+
+/// An error that can occur during sync with a [`CompactFiltersBlockchain`]
+#[derive(Debug)]
+pub enum CompactFiltersError {
+    /// A peer sent an invalid or unexpected response
+    InvalidResponse,
+    /// The headers returned are invalid
+    InvalidHeaders,
+    /// The compact filter headers returned are invalid
+    InvalidFilterHeader,
+    /// The compact filter returned is invalid
+    InvalidFilter,
+    /// The peer is missing a block in the valid chain
+    MissingBlock,
+    /// The data stored in the block filters storage are corrupted
+    DataCorruption,
+
+    /// A peer is not connected
+    NotConnected,
+    /// A peer took too long to reply to one of our messages
+    Timeout,
+
+    /// No peers have been specified
+    NoPeers,
+
+    /// Internal database error
+    DB(rocksdb::Error),
+    /// Internal I/O error
+    IO(std::io::Error),
+    /// Invalid BIP158 filter
+    BIP158(bitcoin::util::bip158::Error),
+    /// Internal system time error
+    Time(std::time::SystemTimeError),
+
+    /// Wrapper for [`crate::error::Error`]
+    Global(Box<crate::error::Error>),
+}
+
+impl fmt::Display for CompactFiltersError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for CompactFiltersError {}
+
+impl_error!(rocksdb::Error, DB, CompactFiltersError);
+impl_error!(std::io::Error, IO, CompactFiltersError);
+impl_error!(bitcoin::util::bip158::Error, BIP158, CompactFiltersError);
+impl_error!(std::time::SystemTimeError, Time, CompactFiltersError);
+
+impl From<crate::error::Error> for CompactFiltersError {
+    fn from(err: crate::error::Error) -> Self {
+        CompactFiltersError::Global(Box::new(err))
+    }
+}

src/blockchain/compact_filters/peer.rs

@@ -0,0 +1,550 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+use std::collections::HashMap;
+use std::net::{TcpStream, ToSocketAddrs};
+use std::sync::{Arc, Condvar, Mutex, RwLock};
+use std::thread;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use socks::{Socks5Stream, ToTargetAddr};
+
+use rand::{thread_rng, Rng};
+
+use bitcoin::consensus::Encodable;
+use bitcoin::hash_types::BlockHash;
+use bitcoin::hashes::Hash;
+use bitcoin::network::constants::ServiceFlags;
+use bitcoin::network::message::{NetworkMessage, RawNetworkMessage};
+use bitcoin::network::message_blockdata::*;
+use bitcoin::network::message_filter::*;
+use bitcoin::network::message_network::VersionMessage;
+use bitcoin::network::stream_reader::StreamReader;
+use bitcoin::network::Address;
+use bitcoin::{Block, Network, Transaction, Txid};
+
+use super::CompactFiltersError;
+
+type ResponsesMap = HashMap<&'static str, Arc<(Mutex<Vec<NetworkMessage>>, Condvar)>>;
+
+pub(crate) const TIMEOUT_SECS: u64 = 30;
+
+/// Container for unconfirmed, but valid Bitcoin transactions
+///
+/// It is normally shared between [`Peer`]s with the use of [`Arc`], so that transactions are not
+/// duplicated in memory.
+#[derive(Debug, Default)]
+pub struct Mempool {
+    txs: RwLock<HashMap<Txid, Transaction>>,
+}
+
+impl Mempool {
+    /// Add a transaction to the mempool
+    ///
+    /// Note that this doesn't propagate the transaction to other
+    /// peers. To do that, [`broadcast`](crate::blockchain::Blockchain::broadcast) should be used.
+    pub fn add_tx(&self, tx: Transaction) {
+        self.txs.write().unwrap().insert(tx.txid(), tx);
+    }
+
+    /// Look-up a transaction in the mempool given an [`Inventory`] request
+    pub fn get_tx(&self, inventory: &Inventory) -> Option<Transaction> {
+        let txid = match inventory {
+            Inventory::Error | Inventory::Block(_) | Inventory::WitnessBlock(_) => return None,
+            Inventory::Transaction(txid) => *txid,
+            Inventory::WitnessTransaction(wtxid) => Txid::from_inner(wtxid.into_inner()),
+        };
+        self.txs.read().unwrap().get(&txid).cloned()
+    }
+
+    /// Return whether or not the mempool contains a transaction with a given txid
+    pub fn has_tx(&self, txid: &Txid) -> bool {
+        self.txs.read().unwrap().contains_key(txid)
+    }
+
+    /// Return the list of transactions contained in the mempool
+    pub fn iter_txs(&self) -> Vec<Transaction> {
+        self.txs.read().unwrap().values().cloned().collect()
+    }
+}
+
+/// A Bitcoin peer
+#[derive(Debug)]
+pub struct Peer {
+    writer: Arc<Mutex<TcpStream>>,
+    responses: Arc<RwLock<ResponsesMap>>,
+
+    reader_thread: thread::JoinHandle<()>,
+    connected: Arc<RwLock<bool>>,
+
+    mempool: Arc<Mempool>,
+
+    version: VersionMessage,
+    network: Network,
+}
+
+impl Peer {
+    /// Connect to a peer over a plaintext TCP connection
+    ///
+    /// This function internally spawns a new thread that will monitor incoming messages from the
+    /// peer, and optionally reply to some of them transparently, like [pings](bitcoin::network::message::NetworkMessage::Ping)
+    pub fn connect<A: ToSocketAddrs>(
+        address: A,
+        mempool: Arc<Mempool>,
+        network: Network,
+    ) -> Result<Self, CompactFiltersError> {
+        let stream = TcpStream::connect(address)?;
+
+        Peer::from_stream(stream, mempool, network)
+    }
+
+    /// Connect to a peer through a SOCKS5 proxy, optionally by using some credentials, specified
+    /// as a tuple of `(username, password)`
+    ///
+    /// This function internally spawns a new thread that will monitor incoming messages from the
+    /// peer, and optionally reply to some of them transparently, like [pings](NetworkMessage::Ping)
+    pub fn connect_proxy<T: ToTargetAddr, P: ToSocketAddrs>(
+        target: T,
+        proxy: P,
+        credentials: Option<(&str, &str)>,
+        mempool: Arc<Mempool>,
+        network: Network,
+    ) -> Result<Self, CompactFiltersError> {
+        let socks_stream = if let Some((username, password)) = credentials {
+            Socks5Stream::connect_with_password(proxy, target, username, password)?
+        } else {
+            Socks5Stream::connect(proxy, target)?
+        };
+
+        Peer::from_stream(socks_stream.into_inner(), mempool, network)
+    }
+
+    /// Create a [`Peer`] from an already connected TcpStream
+    fn from_stream(
+        stream: TcpStream,
+        mempool: Arc<Mempool>,
+        network: Network,
+    ) -> Result<Self, CompactFiltersError> {
+        let writer = Arc::new(Mutex::new(stream.try_clone()?));
+        let responses: Arc<RwLock<ResponsesMap>> = Arc::new(RwLock::new(HashMap::new()));
+        let connected = Arc::new(RwLock::new(true));
+
+        let mut locked_writer = writer.lock().unwrap();
+
+        let reader_thread_responses = Arc::clone(&responses);
+        let reader_thread_writer = Arc::clone(&writer);
+        let reader_thread_mempool = Arc::clone(&mempool);
+        let reader_thread_connected = Arc::clone(&connected);
+        let reader_thread = thread::spawn(move || {
+            Self::reader_thread(
+                network,
+                stream,
+                reader_thread_responses,
+                reader_thread_writer,
+                reader_thread_mempool,
+                reader_thread_connected,
+            )
+        });
+
+        let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() as i64;
+        let nonce = thread_rng().gen();
+        let receiver = Address::new(&locked_writer.peer_addr()?, ServiceFlags::NONE);
+        let sender = Address {
+            services: ServiceFlags::NONE,
+            address: [0u16; 8],
+            port: 0,
+        };
+
+        Self::_send(
+            &mut locked_writer,
+            network.magic(),
+            NetworkMessage::Version(VersionMessage::new(
+                ServiceFlags::WITNESS,
+                timestamp,
+                receiver,
+                sender,
+                nonce,
+                "MagicalBitcoinWallet".into(),
+                0,
+            )),
+        )?;
+        let version = if let NetworkMessage::Version(version) =
+            Self::_recv(&responses, "version", None)?.unwrap()
+        {
+            version
+        } else {
+            return Err(CompactFiltersError::InvalidResponse);
+        };
+
+        if let NetworkMessage::Verack = Self::_recv(&responses, "verack", None)?.unwrap() {
+            Self::_send(&mut locked_writer, network.magic(), NetworkMessage::Verack)?;
+        } else {
+            return Err(CompactFiltersError::InvalidResponse);
+        }
+
+        std::mem::drop(locked_writer);
+
+        Ok(Peer {
+            writer,
+            reader_thread,
+            responses,
+            connected,
+            mempool,
+            network,
+            version,
+        })
+    }
+
+    /// Send a Bitcoin network message
+    fn _send(
+        writer: &mut TcpStream,
+        magic: u32,
+        payload: NetworkMessage,
+    ) -> Result<(), CompactFiltersError> {
+        log::trace!("==> {:?}", payload);
+
+        let raw_message = RawNetworkMessage { magic, payload };
+
+        raw_message
+            .consensus_encode(writer)
+            .map_err(|_| CompactFiltersError::DataCorruption)?;
+
+        Ok(())
+    }
+
+    /// Wait for a specific incoming Bitcoin message, optionally with a timeout
+    fn _recv(
+        responses: &Arc<RwLock<ResponsesMap>>,
+        wait_for: &'static str,
+        timeout: Option<Duration>,
+    ) -> Result<Option<NetworkMessage>, CompactFiltersError> {
+        let message_resp = {
+            let mut lock = responses.write().unwrap();
+            let message_resp = lock.entry(wait_for).or_default();
+            Arc::clone(&message_resp)
+        };
+
+        let (lock, cvar) = &*message_resp;
+
+        let mut messages = lock.lock().unwrap();
+        while messages.is_empty() {
+            match timeout {
+                None => messages = cvar.wait(messages).unwrap(),
+                Some(t) => {
+                    let result = cvar.wait_timeout(messages, t).unwrap();
+                    if result.1.timed_out() {
+                        return Ok(None);
+                    }
+
+                    messages = result.0;
+                }
+            }
+        }
+
+        Ok(messages.pop())
+    }
+
+    /// Return the [`VersionMessage`] sent by the peer
+    pub fn get_version(&self) -> &VersionMessage {
+        &self.version
+    }
+
+    /// Return the Bitcoin [`Network`] in use
+    pub fn get_network(&self) -> Network {
+        self.network
+    }
+
+    /// Return the mempool used by this peer
+    pub fn get_mempool(&self) -> Arc<Mempool> {
+        Arc::clone(&self.mempool)
+    }
+
+    /// Return whether or not the peer is still connected
+    pub fn is_connected(&self) -> bool {
+        *self.connected.read().unwrap()
+    }
+
+    /// Internal function called once the `reader_thread` is spawned
+    fn reader_thread(
+        network: Network,
+        connection: TcpStream,
+        reader_thread_responses: Arc<RwLock<ResponsesMap>>,
+        reader_thread_writer: Arc<Mutex<TcpStream>>,
+        reader_thread_mempool: Arc<Mempool>,
+        reader_thread_connected: Arc<RwLock<bool>>,
+    ) {
+        macro_rules! check_disconnect {
+            ($call:expr) => {
+                match $call {
+                    Ok(good) => good,
+                    Err(e) => {
+                        log::debug!("Error {:?}", e);
+                        *reader_thread_connected.write().unwrap() = false;
+
+                        break;
+                    }
+                }
+            };
+        }
+
+        let mut reader = StreamReader::new(connection, None);
+        loop {
+            let raw_message: RawNetworkMessage = check_disconnect!(reader.read_next());
+
+            let in_message = if raw_message.magic != network.magic() {
+                continue;
+            } else {
+                raw_message.payload
+            };
+
+            log::trace!("<== {:?}", in_message);
+
+            match in_message {
+                NetworkMessage::Ping(nonce) => {
+                    check_disconnect!(Self::_send(
+                        &mut reader_thread_writer.lock().unwrap(),
+                        network.magic(),
+                        NetworkMessage::Pong(nonce),
+                    ));
+
+                    continue;
+                }
+                NetworkMessage::Alert(_) => continue,
+                NetworkMessage::GetData(ref inv) => {
+                    let (found, not_found): (Vec<_>, Vec<_>) = inv
+                        .into_iter()
+                        .map(|item| (*item, reader_thread_mempool.get_tx(item)))
+                        .partition(|(_, d)| d.is_some());
+                    for (_, found_tx) in found {
+                        check_disconnect!(Self::_send(
+                            &mut reader_thread_writer.lock().unwrap(),
+                            network.magic(),
+                            NetworkMessage::Tx(found_tx.unwrap()),
+                        ));
+                    }
+
+                    if !not_found.is_empty() {
+                        check_disconnect!(Self::_send(
+                            &mut reader_thread_writer.lock().unwrap(),
+                            network.magic(),
+                            NetworkMessage::NotFound(
+                                not_found.into_iter().map(|(i, _)| i).collect(),
+                            ),
+                        ));
+                    }
+                }
+                _ => {}
+            }
+
+            let message_resp = {
+                let mut lock = reader_thread_responses.write().unwrap();
+                let message_resp = lock.entry(in_message.cmd()).or_default();
+                Arc::clone(&message_resp)
+            };
+
+            let (lock, cvar) = &*message_resp;
+            let mut messages = lock.lock().unwrap();
+            messages.push(in_message);
+            cvar.notify_all();
+        }
+    }
+
+    /// Send a raw Bitcoin message to the peer
+    pub fn send(&self, payload: NetworkMessage) -> Result<(), CompactFiltersError> {
+        let mut writer = self.writer.lock().unwrap();
+        Self::_send(&mut writer, self.network.magic(), payload)
+    }
+
+    /// Waits for a specific incoming Bitcoin message, optionally with a timeout
+    pub fn recv(
+        &self,
+        wait_for: &'static str,
+        timeout: Option<Duration>,
+    ) -> Result<Option<NetworkMessage>, CompactFiltersError> {
+        Self::_recv(&self.responses, wait_for, timeout)
+    }
+}
+
+pub trait CompactFiltersPeer {
+    fn get_cf_checkpt(
+        &self,
+        filter_type: u8,
+        stop_hash: BlockHash,
+    ) -> Result<CFCheckpt, CompactFiltersError>;
+    fn get_cf_headers(
+        &self,
+        filter_type: u8,
+        start_height: u32,
+        stop_hash: BlockHash,
+    ) -> Result<CFHeaders, CompactFiltersError>;
+    fn get_cf_filters(
+        &self,
+        filter_type: u8,
+        start_height: u32,
+        stop_hash: BlockHash,
+    ) -> Result<(), CompactFiltersError>;
+    fn pop_cf_filter_resp(&self) -> Result<CFilter, CompactFiltersError>;
+}
+
+impl CompactFiltersPeer for Peer {
+    fn get_cf_checkpt(
+        &self,
+        filter_type: u8,
+        stop_hash: BlockHash,
+    ) -> Result<CFCheckpt, CompactFiltersError> {
+        self.send(NetworkMessage::GetCFCheckpt(GetCFCheckpt {
+            filter_type,
+            stop_hash,
+        }))?;
+
+        let response = self
+            .recv("cfcheckpt", Some(Duration::from_secs(TIMEOUT_SECS)))?
+            .ok_or(CompactFiltersError::Timeout)?;
+        let response = match response {
+            NetworkMessage::CFCheckpt(response) => response,
+            _ => return Err(CompactFiltersError::InvalidResponse),
+        };
+
+        if response.filter_type != filter_type {
+            return Err(CompactFiltersError::InvalidResponse);
+        }
+
+        Ok(response)
+    }
+
+    fn get_cf_headers(
+        &self,
+        filter_type: u8,
+        start_height: u32,
+        stop_hash: BlockHash,
+    ) -> Result<CFHeaders, CompactFiltersError> {
+        self.send(NetworkMessage::GetCFHeaders(GetCFHeaders {
+            filter_type,
+            start_height,
+            stop_hash,
+        }))?;
+
+        let response = self
+            .recv("cfheaders", Some(Duration::from_secs(TIMEOUT_SECS)))?
+            .ok_or(CompactFiltersError::Timeout)?;
+        let response = match response {
+            NetworkMessage::CFHeaders(response) => response,
+            _ => return Err(CompactFiltersError::InvalidResponse),
+        };
+
+        if response.filter_type != filter_type {
+            return Err(CompactFiltersError::InvalidResponse);
+        }
+
+        Ok(response)
+    }
+
+    fn pop_cf_filter_resp(&self) -> Result<CFilter, CompactFiltersError> {
+        let response = self
+            .recv("cfilter", Some(Duration::from_secs(TIMEOUT_SECS)))?
+            .ok_or(CompactFiltersError::Timeout)?;
+        let response = match response {
+            NetworkMessage::CFilter(response) => response,
+            _ => return Err(CompactFiltersError::InvalidResponse),
+        };
+
+        Ok(response)
+    }
+
+    fn get_cf_filters(
+        &self,
+        filter_type: u8,
+        start_height: u32,
+        stop_hash: BlockHash,
+    ) -> Result<(), CompactFiltersError> {
+        self.send(NetworkMessage::GetCFilters(GetCFilters {
+            filter_type,
+            start_height,
+            stop_hash,
+        }))?;
+
+        Ok(())
+    }
+}
+
+pub trait InvPeer {
+    fn get_block(&self, block_hash: BlockHash) -> Result<Option<Block>, CompactFiltersError>;
+    fn ask_for_mempool(&self) -> Result<(), CompactFiltersError>;
+    fn broadcast_tx(&self, tx: Transaction) -> Result<(), CompactFiltersError>;
+}
+
+impl InvPeer for Peer {
+    fn get_block(&self, block_hash: BlockHash) -> Result<Option<Block>, CompactFiltersError> {
+        self.send(NetworkMessage::GetData(vec![Inventory::WitnessBlock(
+            block_hash,
+        )]))?;
+
+        match self.recv("block", Some(Duration::from_secs(TIMEOUT_SECS)))? {
+            None => Ok(None),
+            Some(NetworkMessage::Block(response)) => Ok(Some(response)),
+            _ => Err(CompactFiltersError::InvalidResponse),
+        }
+    }
+
+    fn ask_for_mempool(&self) -> Result<(), CompactFiltersError> {
+        self.send(NetworkMessage::MemPool)?;
+        let inv = match self.recv("inv", Some(Duration::from_secs(5)))? {
+            None => return Ok(()), // empty mempool
+            Some(NetworkMessage::Inv(inv)) => inv,
+            _ => return Err(CompactFiltersError::InvalidResponse),
+        };
+
+        let getdata = inv
+            .iter()
+            .cloned()
+            .filter(|item| match item {
+                Inventory::Transaction(txid) if !self.mempool.has_tx(txid) => true,
+                _ => false,
+            })
+            .collect::<Vec<_>>();
+        let num_txs = getdata.len();
+        self.send(NetworkMessage::GetData(getdata))?;
+
+        for _ in 0..num_txs {
+            let tx = self
+                .recv("tx", Some(Duration::from_secs(TIMEOUT_SECS)))?
+                .ok_or(CompactFiltersError::Timeout)?;
+            let tx = match tx {
+                NetworkMessage::Tx(tx) => tx,
+                _ => return Err(CompactFiltersError::InvalidResponse),
+            };
+
+            self.mempool.add_tx(tx);
+        }
+
+        Ok(())
+    }
+
+    fn broadcast_tx(&self, tx: Transaction) -> Result<(), CompactFiltersError> {
+        self.mempool.add_tx(tx.clone());
+        self.send(NetworkMessage::Tx(tx))?;
+
+        Ok(())
+    }
+}

src/blockchain/compact_filters/store.rs

@@ -0,0 +1,904 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+use std::convert::TryInto;
+use std::fmt;
+use std::io::{Read, Write};
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::sync::Arc;
+use std::sync::RwLock;
+
+use rand::distributions::Alphanumeric;
+use rand::{thread_rng, Rng};
+
+use rocksdb::{Direction, IteratorMode, ReadOptions, WriteBatch, DB};
+
+use bitcoin::consensus::{deserialize, encode::VarInt, serialize, Decodable, Encodable};
+use bitcoin::hash_types::FilterHash;
+use bitcoin::hashes::hex::FromHex;
+use bitcoin::hashes::{sha256d, Hash};
+use bitcoin::util::bip158::BlockFilter;
+use bitcoin::util::uint::Uint256;
+use bitcoin::Block;
+use bitcoin::BlockHash;
+use bitcoin::BlockHeader;
+use bitcoin::Network;
+
+use super::CompactFiltersError;
+
+lazy_static! {
+    static ref MAINNET_GENESIS: Block = deserialize(&Vec::<u8>::from_hex("0100000000000000000000000000000000000000000000000000000000000000000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA4B1E5E4A29AB5F49FFFF001D1DAC2B7C0101000000010000000000000000000000000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D0104455468652054696D65732030332F4A616E2F32303039204368616E63656C6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F757420666F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE5548271967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000").unwrap()).unwrap();
+    static ref TESTNET_GENESIS: Block = deserialize(&Vec::<u8>::from_hex("0100000000000000000000000000000000000000000000000000000000000000000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA4B1E5E4ADAE5494DFFFF001D1AA4AE180101000000010000000000000000000000000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D0104455468652054696D65732030332F4A616E2F32303039204368616E63656C6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F757420666F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE5548271967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000").unwrap()).unwrap();
+    static ref REGTEST_GENESIS: Block = deserialize(&Vec::<u8>::from_hex("0100000000000000000000000000000000000000000000000000000000000000000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA4B1E5E4ADAE5494DFFFF7F20020000000101000000010000000000000000000000000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D0104455468652054696D65732030332F4A616E2F32303039204368616E63656C6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F757420666F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE5548271967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000").unwrap()).unwrap();
+}
+
+pub trait StoreType: Default + fmt::Debug {}
+
+#[derive(Default, Debug)]
+pub struct Full;
+impl StoreType for Full {}
+#[derive(Default, Debug)]
+pub struct Snapshot;
+impl StoreType for Snapshot {}
+
+pub enum StoreEntry {
+    BlockHeader(Option<usize>),
+    Block(Option<usize>),
+    BlockHeaderIndex(Option<BlockHash>),
+    CFilterTable((u8, Option<usize>)),
+}
+
+impl StoreEntry {
+    pub fn get_prefix(&self) -> Vec<u8> {
+        match self {
+            StoreEntry::BlockHeader(_) => b"z",
+            StoreEntry::Block(_) => b"x",
+            StoreEntry::BlockHeaderIndex(_) => b"i",
+            StoreEntry::CFilterTable(_) => b"t",
+        }
+        .to_vec()
+    }
+
+    pub fn get_key(&self) -> Vec<u8> {
+        let mut prefix = self.get_prefix();
+        match self {
+            StoreEntry::BlockHeader(Some(height)) => {
+                prefix.extend_from_slice(&height.to_be_bytes())
+            }
+            StoreEntry::Block(Some(height)) => prefix.extend_from_slice(&height.to_be_bytes()),
+            StoreEntry::BlockHeaderIndex(Some(hash)) => {
+                prefix.extend_from_slice(&hash.into_inner())
+            }
+            StoreEntry::CFilterTable((filter_type, bundle_index)) => {
+                prefix.push(*filter_type);
+                if let Some(bundle_index) = bundle_index {
+                    prefix.extend_from_slice(&bundle_index.to_be_bytes());
+                }
+            }
+            _ => {}
+        }
+
+        prefix
+    }
+}
+
+pub trait SerializeDb: Sized {
+    fn serialize(&self) -> Vec<u8>;
+    fn deserialize(data: &[u8]) -> Result<Self, CompactFiltersError>;
+}
+
+impl<T> SerializeDb for T
+where
+    T: Encodable + Decodable,
+{
+    fn serialize(&self) -> Vec<u8> {
+        serialize(self)
+    }
+
+    fn deserialize(data: &[u8]) -> Result<Self, CompactFiltersError> {
+        Ok(deserialize(data).map_err(|_| CompactFiltersError::DataCorruption)?)
+    }
+}
+
+impl Encodable for FilterHeader {
+    fn consensus_encode<W: Write>(
+        &self,
+        mut e: W,
+    ) -> Result<usize, bitcoin::consensus::encode::Error> {
+        let mut written = self.prev_header_hash.consensus_encode(&mut e)?;
+        written += self.filter_hash.consensus_encode(&mut e)?;
+        Ok(written)
+    }
+}
+
+impl Decodable for FilterHeader {
+    fn consensus_decode<D: Read>(mut d: D) -> Result<Self, bitcoin::consensus::encode::Error> {
+        let prev_header_hash = FilterHeaderHash::consensus_decode(&mut d)?;
+        let filter_hash = FilterHash::consensus_decode(&mut d)?;
+
+        Ok(FilterHeader {
+            prev_header_hash,
+            filter_hash,
+        })
+    }
+}
+
+impl Encodable for BundleStatus {
+    fn consensus_encode<W: Write>(
+        &self,
+        mut e: W,
+    ) -> Result<usize, bitcoin::consensus::encode::Error> {
+        let mut written = 0;
+
+        match self {
+            BundleStatus::Init => {
+                written += 0x00u8.consensus_encode(&mut e)?;
+            }
+            BundleStatus::CFHeaders { cf_headers } => {
+                written += 0x01u8.consensus_encode(&mut e)?;
+                written += VarInt(cf_headers.len() as u64).consensus_encode(&mut e)?;
+                for header in cf_headers {
+                    written += header.consensus_encode(&mut e)?;
+                }
+            }
+            BundleStatus::CFilters { cf_filters } => {
+                written += 0x02u8.consensus_encode(&mut e)?;
+                written += VarInt(cf_filters.len() as u64).consensus_encode(&mut e)?;
+                for filter in cf_filters {
+                    written += filter.consensus_encode(&mut e)?;
+                }
+            }
+            BundleStatus::Processed { cf_filters } => {
+                written += 0x03u8.consensus_encode(&mut e)?;
+                written += VarInt(cf_filters.len() as u64).consensus_encode(&mut e)?;
+                for filter in cf_filters {
+                    written += filter.consensus_encode(&mut e)?;
+                }
+            }
+            BundleStatus::Pruned => {
+                written += 0x04u8.consensus_encode(&mut e)?;
+            }
+            BundleStatus::Tip { cf_filters } => {
+                written += 0x05u8.consensus_encode(&mut e)?;
+                written += VarInt(cf_filters.len() as u64).consensus_encode(&mut e)?;
+                for filter in cf_filters {
+                    written += filter.consensus_encode(&mut e)?;
+                }
+            }
+        }
+
+        Ok(written)
+    }
+}
+
+impl Decodable for BundleStatus {
+    fn consensus_decode<D: Read>(mut d: D) -> Result<Self, bitcoin::consensus::encode::Error> {
+        let byte_type = u8::consensus_decode(&mut d)?;
+        match byte_type {
+            0x00 => Ok(BundleStatus::Init),
+            0x01 => {
+                let num = VarInt::consensus_decode(&mut d)?;
+                let num = num.0 as usize;
+
+                let mut cf_headers = Vec::with_capacity(num);
+                for _ in 0..num {
+                    cf_headers.push(FilterHeader::consensus_decode(&mut d)?);
+                }
+
+                Ok(BundleStatus::CFHeaders { cf_headers })
+            }
+            0x02 => {
+                let num = VarInt::consensus_decode(&mut d)?;
+                let num = num.0 as usize;
+
+                let mut cf_filters = Vec::with_capacity(num);
+                for _ in 0..num {
+                    cf_filters.push(Vec::<u8>::consensus_decode(&mut d)?);
+                }
+
+                Ok(BundleStatus::CFilters { cf_filters })
+            }
+            0x03 => {
+                let num = VarInt::consensus_decode(&mut d)?;
+                let num = num.0 as usize;
+
+                let mut cf_filters = Vec::with_capacity(num);
+                for _ in 0..num {
+                    cf_filters.push(Vec::<u8>::consensus_decode(&mut d)?);
+                }
+
+                Ok(BundleStatus::Processed { cf_filters })
+            }
+            0x04 => Ok(BundleStatus::Pruned),
+            0x05 => {
+                let num = VarInt::consensus_decode(&mut d)?;
+                let num = num.0 as usize;
+
+                let mut cf_filters = Vec::with_capacity(num);
+                for _ in 0..num {
+                    cf_filters.push(Vec::<u8>::consensus_decode(&mut d)?);
+                }
+
+                Ok(BundleStatus::Tip { cf_filters })
+            }
+            _ => Err(bitcoin::consensus::encode::Error::ParseFailed(
+                "Invalid byte type",
+            )),
+        }
+    }
+}
+
+pub struct ChainStore<T: StoreType> {
+    store: Arc<RwLock<DB>>,
+    cf_name: String,
+    min_height: usize,
+    network: Network,
+    phantom: PhantomData<T>,
+}
+
+impl ChainStore<Full> {
+    pub fn new(store: DB, network: Network) -> Result<Self, CompactFiltersError> {
+        let genesis = match network {
+            Network::Bitcoin => MAINNET_GENESIS.deref(),
+            Network::Testnet => TESTNET_GENESIS.deref(),
+            Network::Regtest => REGTEST_GENESIS.deref(),
+        };
+
+        let cf_name = "default".to_string();
+        let cf_handle = store.cf_handle(&cf_name).unwrap();
+
+        let genesis_key = StoreEntry::BlockHeader(Some(0)).get_key();
+
+        if store.get_pinned_cf(cf_handle, &genesis_key)?.is_none() {
+            let mut batch = WriteBatch::default();
+            batch.put_cf(
+                cf_handle,
+                genesis_key,
+                (genesis.header, genesis.header.work()).serialize(),
+            );
+            batch.put_cf(
+                cf_handle,
+                StoreEntry::BlockHeaderIndex(Some(genesis.block_hash())).get_key(),
+                &0usize.to_be_bytes(),
+            );
+            store.write(batch)?;
+        }
+
+        Ok(ChainStore {
+            store: Arc::new(RwLock::new(store)),
+            cf_name,
+            min_height: 0,
+            network,
+            phantom: PhantomData,
+        })
+    }
+
+    pub fn get_locators(&self) -> Result<Vec<(BlockHash, usize)>, CompactFiltersError> {
+        let mut step = 1;
+        let mut index = self.get_height()?;
+        let mut answer = Vec::new();
+
+        let store_read = self.store.read().unwrap();
+        let cf_handle = store_read.cf_handle(&self.cf_name).unwrap();
+
+        loop {
+            if answer.len() > 10 {
+                step *= 2;
+            }
+
+            let (header, _): (BlockHeader, Uint256) = SerializeDb::deserialize(
+                &store_read
+                    .get_pinned_cf(cf_handle, StoreEntry::BlockHeader(Some(index)).get_key())?
+                    .unwrap(),
+            )?;
+            answer.push((header.block_hash(), index));
+
+            if let Some(new_index) = index.checked_sub(step) {
+                index = new_index;
+            } else {
+                break;
+            }
+        }
+
+        Ok(answer)
+    }
+
+    pub fn start_snapshot(&self, from: usize) -> Result<ChainStore<Snapshot>, CompactFiltersError> {
+        let new_cf_name: String = thread_rng().sample_iter(&Alphanumeric).take(16).collect();
+        let new_cf_name = format!("_headers:{}", new_cf_name);
+
+        let mut write_store = self.store.write().unwrap();
+
+        write_store.create_cf(&new_cf_name, &Default::default())?;
+
+        let cf_handle = write_store.cf_handle(&self.cf_name).unwrap();
+        let new_cf_handle = write_store.cf_handle(&new_cf_name).unwrap();
+
+        let (header, work): (BlockHeader, Uint256) = SerializeDb::deserialize(
+            &write_store
+                .get_pinned_cf(cf_handle, StoreEntry::BlockHeader(Some(from)).get_key())?
+                .ok_or(CompactFiltersError::DataCorruption)?,
+        )?;
+
+        let mut batch = WriteBatch::default();
+        batch.put_cf(
+            new_cf_handle,
+            StoreEntry::BlockHeaderIndex(Some(header.block_hash())).get_key(),
+            &from.to_be_bytes(),
+        );
+        batch.put_cf(
+            new_cf_handle,
+            StoreEntry::BlockHeader(Some(from)).get_key(),
+            (header, work).serialize(),
+        );
+        write_store.write(batch)?;
+
+        let store = Arc::clone(&self.store);
+        Ok(ChainStore {
+            store,
+            cf_name: new_cf_name,
+            min_height: from,
+            network: self.network,
+            phantom: PhantomData,
+        })
+    }
+
+    pub fn recover_snapshot(&self, cf_name: &str) -> Result<(), CompactFiltersError> {
+        let mut write_store = self.store.write().unwrap();
+        let snapshot_cf_handle = write_store.cf_handle(cf_name).unwrap();
+
+        let prefix = StoreEntry::BlockHeader(None).get_key();
+        let mut iterator = write_store.prefix_iterator_cf(snapshot_cf_handle, prefix);
+
+        let min_height = match iterator
+            .next()
+            .and_then(|(k, _)| k[1..].try_into().ok())
+            .map(|bytes| usize::from_be_bytes(bytes))
+        {
+            None => {
+                std::mem::drop(iterator);
+                write_store.drop_cf(cf_name).ok();
+
+                return Ok(());
+            }
+            Some(x) => x,
+        };
+        std::mem::drop(iterator);
+        std::mem::drop(write_store);
+
+        let snapshot = ChainStore {
+            store: Arc::clone(&self.store),
+            cf_name: cf_name.into(),
+            min_height,
+            network: self.network,
+            phantom: PhantomData,
+        };
+        if snapshot.work()? > self.work()? {
+            self.apply_snapshot(snapshot)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn apply_snapshot(
+        &self,
+        snaphost: ChainStore<Snapshot>,
+    ) -> Result<(), CompactFiltersError> {
+        let mut batch = WriteBatch::default();
+
+        let read_store = self.store.read().unwrap();
+        let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
+        let snapshot_cf_handle = read_store.cf_handle(&snaphost.cf_name).unwrap();
+
+        let from_key = StoreEntry::BlockHeader(Some(snaphost.min_height)).get_key();
+        let to_key = StoreEntry::BlockHeader(Some(usize::MAX)).get_key();
+
+        let mut opts = ReadOptions::default();
+        opts.set_iterate_upper_bound(to_key.clone());
+
+        log::debug!("Removing items");
+        batch.delete_range_cf(cf_handle, &from_key, &to_key);
+        for (_, v) in read_store.iterator_cf_opt(
+            cf_handle,
+            opts,
+            IteratorMode::From(&from_key, Direction::Forward),
+        ) {
+            let (header, _): (BlockHeader, Uint256) = SerializeDb::deserialize(&v)?;
+
+            batch.delete_cf(
+                cf_handle,
+                StoreEntry::BlockHeaderIndex(Some(header.block_hash())).get_key(),
+            );
+        }
+
+        // Delete full blocks overriden by snapshot
+        let from_key = StoreEntry::Block(Some(snaphost.min_height)).get_key();
+        let to_key = StoreEntry::Block(Some(usize::MAX)).get_key();
+        batch.delete_range(&from_key, &to_key);
+
+        log::debug!("Copying over new items");
+        for (k, v) in read_store.iterator_cf(snapshot_cf_handle, IteratorMode::Start) {
+            batch.put_cf(cf_handle, k, v);
+        }
+
+        read_store.write(batch)?;
+
+        std::mem::drop(snapshot_cf_handle);
+        std::mem::drop(cf_handle);
+        std::mem::drop(read_store);
+
+        self.store.write().unwrap().drop_cf(&snaphost.cf_name)?;
+
+        Ok(())
+    }
+
+    pub fn get_height_for(
+        &self,
+        block_hash: &BlockHash,
+    ) -> Result<Option<usize>, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+        let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
+
+        let key = StoreEntry::BlockHeaderIndex(Some(block_hash.clone())).get_key();
+        let data = read_store.get_pinned_cf(cf_handle, key)?;
+        Ok(data
+            .map(|data| {
+                Ok::<_, CompactFiltersError>(usize::from_be_bytes(
+                    data.as_ref()
+                        .try_into()
+                        .map_err(|_| CompactFiltersError::DataCorruption)?,
+                ))
+            })
+            .transpose()?)
+    }
+
+    pub fn get_block_hash(&self, height: usize) -> Result<Option<BlockHash>, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+        let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
+
+        let key = StoreEntry::BlockHeader(Some(height)).get_key();
+        let data = read_store.get_pinned_cf(cf_handle, key)?;
+        Ok(data
+            .map(|data| {
+                let (header, _): (BlockHeader, Uint256) =
+                    deserialize(&data).map_err(|_| CompactFiltersError::DataCorruption)?;
+                Ok::<_, CompactFiltersError>(header.block_hash())
+            })
+            .transpose()?)
+    }
+
+    pub fn save_full_block(&self, block: &Block, height: usize) -> Result<(), CompactFiltersError> {
+        let key = StoreEntry::Block(Some(height)).get_key();
+        self.store.read().unwrap().put(key, block.serialize())?;
+
+        Ok(())
+    }
+
+    pub fn get_full_block(&self, height: usize) -> Result<Option<Block>, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+
+        let key = StoreEntry::Block(Some(height)).get_key();
+        let opt_block = read_store.get_pinned(key)?;
+
+        Ok(opt_block
+            .map(|data| deserialize(&data))
+            .transpose()
+            .map_err(|_| CompactFiltersError::DataCorruption)?)
+    }
+
+    pub fn delete_blocks_until(&self, height: usize) -> Result<(), CompactFiltersError> {
+        let from_key = StoreEntry::Block(Some(0)).get_key();
+        let to_key = StoreEntry::Block(Some(height)).get_key();
+
+        let mut batch = WriteBatch::default();
+        batch.delete_range(&from_key, &to_key);
+
+        self.store.read().unwrap().write(batch)?;
+
+        Ok(())
+    }
+
+    pub fn iter_full_blocks(&self) -> Result<Vec<(usize, Block)>, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+
+        let prefix = StoreEntry::Block(None).get_key();
+
+        let iterator = read_store.prefix_iterator(&prefix);
+        // FIXME: we have to filter manually because rocksdb sometimes returns stuff that doesn't
+        // have the right prefix
+        iterator
+            .filter(|(k, _)| k.starts_with(&prefix))
+            .map(|(k, v)| {
+                let height: usize = usize::from_be_bytes(
+                    k[1..]
+                        .try_into()
+                        .map_err(|_| CompactFiltersError::DataCorruption)?,
+                );
+                let block = SerializeDb::deserialize(&v)?;
+
+                Ok((height, block))
+            })
+            .collect::<Result<_, _>>()
+    }
+}
+
+impl<T: StoreType> ChainStore<T> {
+    pub fn work(&self) -> Result<Uint256, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+        let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
+
+        let prefix = StoreEntry::BlockHeader(None).get_key();
+        let iterator = read_store.prefix_iterator_cf(cf_handle, prefix);
+
+        Ok(iterator
+            .last()
+            .map(|(_, v)| -> Result<_, CompactFiltersError> {
+                let (_, work): (BlockHeader, Uint256) = SerializeDb::deserialize(&v)?;
+
+                Ok(work)
+            })
+            .transpose()?
+            .unwrap_or_default())
+    }
+
+    pub fn get_height(&self) -> Result<usize, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+        let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
+
+        let prefix = StoreEntry::BlockHeader(None).get_key();
+        let iterator = read_store.prefix_iterator_cf(cf_handle, prefix);
+
+        Ok(iterator
+            .last()
+            .map(|(k, _)| -> Result<_, CompactFiltersError> {
+                let height = usize::from_be_bytes(
+                    k[1..]
+                        .try_into()
+                        .map_err(|_| CompactFiltersError::DataCorruption)?,
+                );
+
+                Ok(height)
+            })
+            .transpose()?
+            .unwrap_or_default())
+    }
+
+    pub fn get_tip_hash(&self) -> Result<Option<BlockHash>, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+        let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
+
+        let prefix = StoreEntry::BlockHeader(None).get_key();
+        let iterator = read_store.prefix_iterator_cf(cf_handle, prefix);
+
+        Ok(iterator
+            .last()
+            .map(|(_, v)| -> Result<_, CompactFiltersError> {
+                let (header, _): (BlockHeader, Uint256) = SerializeDb::deserialize(&v)?;
+
+                Ok(header.block_hash())
+            })
+            .transpose()?)
+    }
+
+    pub fn apply(
+        &mut self,
+        from: usize,
+        headers: Vec<BlockHeader>,
+    ) -> Result<BlockHash, CompactFiltersError> {
+        let mut batch = WriteBatch::default();
+
+        let read_store = self.store.read().unwrap();
+        let cf_handle = read_store.cf_handle(&self.cf_name).unwrap();
+
+        let (mut last_hash, mut accumulated_work) = read_store
+            .get_pinned_cf(cf_handle, StoreEntry::BlockHeader(Some(from)).get_key())?
+            .map(|result| {
+                let (header, work): (BlockHeader, Uint256) = SerializeDb::deserialize(&result)?;
+                Ok::<_, CompactFiltersError>((header.block_hash(), work))
+            })
+            .transpose()?
+            .ok_or(CompactFiltersError::DataCorruption)?;
+
+        for (index, header) in headers.into_iter().enumerate() {
+            if header.prev_blockhash != last_hash {
+                return Err(CompactFiltersError::InvalidHeaders);
+            }
+
+            last_hash = header.block_hash();
+            accumulated_work = accumulated_work + header.work();
+
+            let height = from + index + 1;
+            batch.put_cf(
+                cf_handle,
+                StoreEntry::BlockHeaderIndex(Some(header.block_hash())).get_key(),
+                &(height).to_be_bytes(),
+            );
+            batch.put_cf(
+                cf_handle,
+                StoreEntry::BlockHeader(Some(height)).get_key(),
+                (header, accumulated_work).serialize(),
+            );
+        }
+
+        std::mem::drop(cf_handle);
+        std::mem::drop(read_store);
+
+        self.store.write().unwrap().write(batch)?;
+        Ok(last_hash)
+    }
+}
+
+impl<T: StoreType> fmt::Debug for ChainStore<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct(&format!("ChainStore<{:?}>", T::default()))
+            .field("cf_name", &self.cf_name)
+            .field("min_height", &self.min_height)
+            .field("network", &self.network)
+            .field("headers_height", &self.get_height())
+            .field("tip_hash", &self.get_tip_hash())
+            .finish()
+    }
+}
+
+pub type FilterHeaderHash = FilterHash;
+
+#[derive(Debug, Clone)]
+pub struct FilterHeader {
+    prev_header_hash: FilterHeaderHash,
+    filter_hash: FilterHash,
+}
+
+impl FilterHeader {
+    fn header_hash(&self) -> FilterHeaderHash {
+        let mut hash_data = self.filter_hash.into_inner().to_vec();
+        hash_data.extend_from_slice(&self.prev_header_hash);
+        sha256d::Hash::hash(&hash_data).into()
+    }
+}
+
+pub enum BundleStatus {
+    Init,
+    CFHeaders { cf_headers: Vec<FilterHeader> },
+    CFilters { cf_filters: Vec<Vec<u8>> },
+    Processed { cf_filters: Vec<Vec<u8>> },
+    Tip { cf_filters: Vec<Vec<u8>> },
+    Pruned,
+}
+
+pub struct CFStore {
+    store: Arc<RwLock<DB>>,
+    filter_type: u8,
+}
+
+type BundleEntry = (BundleStatus, FilterHeaderHash);
+
+impl CFStore {
+    pub fn new(
+        headers_store: &ChainStore<Full>,
+        filter_type: u8,
+    ) -> Result<Self, CompactFiltersError> {
+        let cf_store = CFStore {
+            store: Arc::clone(&headers_store.store),
+            filter_type,
+        };
+
+        let genesis = match headers_store.network {
+            Network::Bitcoin => MAINNET_GENESIS.deref(),
+            Network::Testnet => TESTNET_GENESIS.deref(),
+            Network::Regtest => REGTEST_GENESIS.deref(),
+        };
+
+        let filter = BlockFilter::new_script_filter(genesis, |utxo| {
+            Err(bitcoin::util::bip158::Error::UtxoMissing(*utxo))
+        })?;
+        let first_key = StoreEntry::CFilterTable((filter_type, Some(0))).get_key();
+
+        // Add the genesis' filter
+        {
+            let read_store = cf_store.store.read().unwrap();
+            if read_store.get_pinned(&first_key)?.is_none() {
+                read_store.put(
+                    &first_key,
+                    (BundleStatus::Init, filter.filter_id(&FilterHash::default())).serialize(),
+                )?;
+            }
+        }
+
+        Ok(cf_store)
+    }
+
+    pub fn get_filter_type(&self) -> u8 {
+        self.filter_type
+    }
+
+    pub fn get_bundles(&self) -> Result<Vec<BundleEntry>, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+
+        let prefix = StoreEntry::CFilterTable((self.filter_type, None)).get_key();
+        let iterator = read_store.prefix_iterator(&prefix);
+
+        // FIXME: we have to filter manually because rocksdb sometimes returns stuff that doesn't
+        // have the right prefix
+        iterator
+            .filter(|(k, _)| k.starts_with(&prefix))
+            .map(|(_, data)| BundleEntry::deserialize(&data))
+            .collect::<Result<_, _>>()
+    }
+
+    pub fn get_checkpoints(&self) -> Result<Vec<FilterHash>, CompactFiltersError> {
+        let read_store = self.store.read().unwrap();
+
+        let prefix = StoreEntry::CFilterTable((self.filter_type, None)).get_key();
+        let iterator = read_store.prefix_iterator(&prefix);
+
+        // FIXME: we have to filter manually because rocksdb sometimes returns stuff that doesn't
+        // have the right prefix
+        Ok(iterator
+            .filter(|(k, _)| k.starts_with(&prefix))
+            .skip(1)
+            .map(|(_, data)| Ok::<_, CompactFiltersError>(BundleEntry::deserialize(&data)?.1))
+            .collect::<Result<_, _>>()?)
+    }
+
+    pub fn replace_checkpoints(
+        &self,
+        checkpoints: Vec<FilterHash>,
+    ) -> Result<(), CompactFiltersError> {
+        let current_checkpoints = self.get_checkpoints()?;
+
+        let mut equal_bundles = 0;
+        for (index, (our, their)) in current_checkpoints
+            .iter()
+            .zip(checkpoints.iter())
+            .enumerate()
+        {
+            equal_bundles = index;
+
+            if our != their {
+                break;
+            }
+        }
+
+        let read_store = self.store.read().unwrap();
+        let mut batch = WriteBatch::default();
+
+        for (index, filter_hash) in checkpoints.iter().enumerate().skip(equal_bundles) {
+            let key = StoreEntry::CFilterTable((self.filter_type, Some(index + 1))).get_key(); // +1 to skip the genesis' filter
+
+            if let Some((BundleStatus::Tip { .. }, _)) = read_store
+                .get_pinned(&key)?
+                .map(|data| BundleEntry::deserialize(&data))
+                .transpose()?
+            {
+                println!("Keeping bundle #{} as Tip", index);
+            } else {
+                batch.put(&key, (BundleStatus::Init, *filter_hash).serialize());
+            }
+        }
+
+        read_store.write(batch)?;
+
+        Ok(())
+    }
+
+    pub fn advance_to_cf_headers(
+        &self,
+        bundle: usize,
+        checkpoint_hash: FilterHeaderHash,
+        filter_headers: Vec<FilterHash>,
+    ) -> Result<BundleStatus, CompactFiltersError> {
+        let mut last_hash = checkpoint_hash;
+        let cf_headers = filter_headers
+            .into_iter()
+            .map(|filter_hash| {
+                let filter_header = FilterHeader {
+                    prev_header_hash: last_hash,
+                    filter_hash,
+                };
+                last_hash = filter_header.header_hash();
+
+                filter_header
+            })
+            .collect();
+
+        let read_store = self.store.read().unwrap();
+
+        let next_key = StoreEntry::CFilterTable((self.filter_type, Some(bundle + 1))).get_key(); // +1 to skip the genesis' filter
+        if let Some((_, next_checkpoint)) = read_store
+            .get_pinned(&next_key)?
+            .map(|data| BundleEntry::deserialize(&data))
+            .transpose()?
+        {
+            // check connection with the next bundle if present
+            if last_hash != next_checkpoint {
+                return Err(CompactFiltersError::InvalidFilterHeader);
+            }
+        }
+
+        let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
+        let value = (BundleStatus::CFHeaders { cf_headers }, checkpoint_hash);
+
+        read_store.put(key, value.serialize())?;
+
+        Ok(value.0)
+    }
+
+    pub fn advance_to_cf_filters(
+        &self,
+        bundle: usize,
+        checkpoint_hash: FilterHeaderHash,
+        headers: Vec<FilterHeader>,
+        filters: Vec<(usize, Vec<u8>)>,
+    ) -> Result<BundleStatus, CompactFiltersError> {
+        let cf_filters = filters
+            .into_iter()
+            .zip(headers.iter())
+            .map(|((_, filter_content), header)| {
+                if header.filter_hash != sha256d::Hash::hash(&filter_content).into() {
+                    return Err(CompactFiltersError::InvalidFilter);
+                }
+
+                Ok::<_, CompactFiltersError>(filter_content)
+            })
+            .collect::<Result<_, _>>()?;
+
+        let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
+        let value = (BundleStatus::CFilters { cf_filters }, checkpoint_hash);
+
+        let read_store = self.store.read().unwrap();
+        read_store.put(key, value.serialize())?;
+
+        Ok(value.0)
+    }
+
+    pub fn prune_filters(
+        &self,
+        bundle: usize,
+        checkpoint_hash: FilterHeaderHash,
+    ) -> Result<BundleStatus, CompactFiltersError> {
+        let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
+        let value = (BundleStatus::Pruned, checkpoint_hash);
+
+        let read_store = self.store.read().unwrap();
+        read_store.put(key, value.serialize())?;
+
+        Ok(value.0)
+    }
+
+    pub fn mark_as_tip(
+        &self,
+        bundle: usize,
+        cf_filters: Vec<Vec<u8>>,
+        checkpoint_hash: FilterHeaderHash,
+    ) -> Result<BundleStatus, CompactFiltersError> {
+        let key = StoreEntry::CFilterTable((self.filter_type, Some(bundle))).get_key();
+        let value = (BundleStatus::Tip { cf_filters }, checkpoint_hash);
+
+        let read_store = self.store.read().unwrap();
+        read_store.put(key, value.serialize())?;
+
+        Ok(value.0)
+    }
+}

src/blockchain/compact_filters/sync.rs

@@ -0,0 +1,313 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+use std::collections::{BTreeMap, HashMap, VecDeque};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use bitcoin::hash_types::{BlockHash, FilterHash};
+use bitcoin::network::message::NetworkMessage;
+use bitcoin::network::message_blockdata::GetHeadersMessage;
+use bitcoin::util::bip158::BlockFilter;
+
+use super::peer::*;
+use super::store::*;
+use super::CompactFiltersError;
+use crate::error::Error;
+
+pub(crate) const BURIED_CONFIRMATIONS: usize = 100;
+
+pub struct CFSync {
+    headers_store: Arc<ChainStore<Full>>,
+    cf_store: Arc<CFStore>,
+    skip_blocks: usize,
+    bundles: Mutex<VecDeque<(BundleStatus, FilterHash, usize)>>,
+}
+
+impl CFSync {
+    pub fn new(
+        headers_store: Arc<ChainStore<Full>>,
+        skip_blocks: usize,
+        filter_type: u8,
+    ) -> Result<Self, CompactFiltersError> {
+        let cf_store = Arc::new(CFStore::new(&headers_store, filter_type)?);
+
+        Ok(CFSync {
+            headers_store,
+            cf_store,
+            skip_blocks,
+            bundles: Mutex::new(VecDeque::new()),
+        })
+    }
+
+    pub fn pruned_bundles(&self) -> Result<usize, CompactFiltersError> {
+        Ok(self
+            .cf_store
+            .get_bundles()?
+            .into_iter()
+            .skip(self.skip_blocks / 1000)
+            .fold(0, |acc, (status, _)| match status {
+                BundleStatus::Pruned => acc + 1,
+                _ => acc,
+            }))
+    }
+
+    pub fn prepare_sync(&self, peer: Arc<Peer>) -> Result<(), CompactFiltersError> {
+        let mut bundles_lock = self.bundles.lock().unwrap();
+
+        let resp = peer.get_cf_checkpt(
+            self.cf_store.get_filter_type(),
+            self.headers_store.get_tip_hash()?.unwrap(),
+        )?;
+        self.cf_store.replace_checkpoints(resp.filter_headers)?;
+
+        bundles_lock.clear();
+        for (index, (status, checkpoint)) in self.cf_store.get_bundles()?.into_iter().enumerate() {
+            bundles_lock.push_back((status, checkpoint, index));
+        }
+
+        Ok(())
+    }
+
+    pub fn capture_thread_for_sync<F, Q>(
+        &self,
+        peer: Arc<Peer>,
+        process: F,
+        completed_bundle: Q,
+    ) -> Result<(), CompactFiltersError>
+    where
+        F: Fn(&BlockHash, &BlockFilter) -> Result<bool, CompactFiltersError>,
+        Q: Fn(usize) -> Result<(), Error>,
+    {
+        let current_height = self.headers_store.get_height()?; // TODO: we should update it in case headers_store is also updated
+
+        loop {
+            let (mut status, checkpoint, index) = match self.bundles.lock().unwrap().pop_front() {
+                None => break,
+                Some(x) => x,
+            };
+
+            log::debug!(
+                "Processing bundle #{} - height {} to {}",
+                index,
+                index * 1000 + 1,
+                (index + 1) * 1000
+            );
+
+            let process_received_filters =
+                |expected_filters| -> Result<BTreeMap<usize, Vec<u8>>, CompactFiltersError> {
+                    let mut filters_map = BTreeMap::new();
+                    for _ in 0..expected_filters {
+                        let filter = peer.pop_cf_filter_resp()?;
+                        if filter.filter_type != self.cf_store.get_filter_type() {
+                            return Err(CompactFiltersError::InvalidResponse);
+                        }
+
+                        match self.headers_store.get_height_for(&filter.block_hash)? {
+                            Some(height) => filters_map.insert(height, filter.filter),
+                            None => return Err(CompactFiltersError::InvalidFilter),
+                        };
+                    }
+
+                    Ok(filters_map)
+                };
+
+            let start_height = index * 1000 + 1;
+            let mut already_processed = 0;
+
+            if start_height < self.skip_blocks {
+                status = self.cf_store.prune_filters(index, checkpoint)?;
+            }
+
+            let stop_height = std::cmp::min(current_height, start_height + 999);
+            let stop_hash = self.headers_store.get_block_hash(stop_height)?.unwrap();
+
+            if let BundleStatus::Init = status {
+                log::trace!("status: Init");
+
+                let resp = peer.get_cf_headers(0x00, start_height as u32, stop_hash)?;
+
+                assert!(resp.previous_filter == checkpoint);
+                status =
+                    self.cf_store
+                        .advance_to_cf_headers(index, checkpoint, resp.filter_hashes)?;
+            }
+            if let BundleStatus::Tip { cf_filters } = status {
+                log::trace!("status: Tip (beginning) ");
+
+                already_processed = cf_filters.len();
+                let headers_resp = peer.get_cf_headers(0x00, start_height as u32, stop_hash)?;
+
+                let cf_headers = match self.cf_store.advance_to_cf_headers(
+                    index,
+                    checkpoint,
+                    headers_resp.filter_hashes,
+                )? {
+                    BundleStatus::CFHeaders { cf_headers } => cf_headers,
+                    _ => return Err(CompactFiltersError::InvalidResponse),
+                };
+
+                peer.get_cf_filters(
+                    self.cf_store.get_filter_type(),
+                    (start_height + cf_filters.len()) as u32,
+                    stop_hash,
+                )?;
+                let expected_filters = stop_height - start_height + 1 - cf_filters.len();
+                let filters_map = process_received_filters(expected_filters)?;
+                let filters = cf_filters
+                    .into_iter()
+                    .enumerate()
+                    .chain(filters_map.into_iter())
+                    .collect();
+                status = self
+                    .cf_store
+                    .advance_to_cf_filters(index, checkpoint, cf_headers, filters)?;
+            }
+            if let BundleStatus::CFHeaders { cf_headers } = status {
+                log::trace!("status: CFHeaders");
+
+                peer.get_cf_filters(
+                    self.cf_store.get_filter_type(),
+                    start_height as u32,
+                    stop_hash,
+                )?;
+                let expected_filters = stop_height - start_height + 1;
+                let filters_map = process_received_filters(expected_filters)?;
+                status = self.cf_store.advance_to_cf_filters(
+                    index,
+                    checkpoint,
+                    cf_headers,
+                    filters_map.into_iter().collect(),
+                )?;
+            }
+            if let BundleStatus::CFilters { cf_filters } = status {
+                log::trace!("status: CFilters");
+
+                let last_sync_buried_height = (start_height + already_processed)
+                    .checked_sub(BURIED_CONFIRMATIONS)
+                    .unwrap_or(0);
+
+                for (filter_index, filter) in cf_filters.iter().enumerate() {
+                    let height = filter_index + start_height;
+
+                    // do not download blocks that were already "buried" since the last sync
+                    if height < last_sync_buried_height {
+                        continue;
+                    }
+
+                    let block_hash = self.headers_store.get_block_hash(height)?.unwrap();
+
+                    // TODO: also download random blocks?
+                    if process(&block_hash, &BlockFilter::new(&filter))? {
+                        log::debug!("Downloading block {}", block_hash);
+
+                        let block = peer
+                            .get_block(block_hash)?
+                            .ok_or(CompactFiltersError::MissingBlock)?;
+                        self.headers_store.save_full_block(&block, height)?;
+                    }
+                }
+
+                status = BundleStatus::Processed { cf_filters };
+            }
+            if let BundleStatus::Processed { cf_filters } = status {
+                log::trace!("status: Processed");
+
+                if current_height - stop_height > 1000 {
+                    status = self.cf_store.prune_filters(index, checkpoint)?;
+                } else {
+                    status = self.cf_store.mark_as_tip(index, cf_filters, checkpoint)?;
+                }
+
+                completed_bundle(index)?;
+            }
+            if let BundleStatus::Pruned = status {
+                log::trace!("status: Pruned");
+            }
+            if let BundleStatus::Tip { .. } = status {
+                log::trace!("status: Tip");
+            }
+        }
+
+        Ok(())
+    }
+}
+
+pub fn sync_headers<F>(
+    peer: Arc<Peer>,
+    store: Arc<ChainStore<Full>>,
+    sync_fn: F,
+) -> Result<Option<ChainStore<Snapshot>>, CompactFiltersError>
+where
+    F: Fn(usize) -> Result<(), Error>,
+{
+    let locators = store.get_locators()?;
+    let locators_vec = locators.iter().map(|(hash, _)| hash).cloned().collect();
+    let locators_map: HashMap<_, _> = locators.into_iter().collect();
+
+    peer.send(NetworkMessage::GetHeaders(GetHeadersMessage::new(
+        locators_vec,
+        Default::default(),
+    )))?;
+    let (mut snapshot, mut last_hash) = if let NetworkMessage::Headers(headers) = peer
+        .recv("headers", Some(Duration::from_secs(TIMEOUT_SECS)))?
+        .ok_or(CompactFiltersError::Timeout)?
+    {
+        if headers.is_empty() {
+            return Ok(None);
+        }
+
+        match locators_map.get(&headers[0].prev_blockhash) {
+            None => return Err(CompactFiltersError::InvalidHeaders),
+            Some(from) => (
+                store.start_snapshot(*from)?,
+                headers[0].prev_blockhash.clone(),
+            ),
+        }
+    } else {
+        return Err(CompactFiltersError::InvalidResponse);
+    };
+
+    let mut sync_height = store.get_height()?;
+    while sync_height < peer.get_version().start_height as usize {
+        peer.send(NetworkMessage::GetHeaders(GetHeadersMessage::new(
+            vec![last_hash],
+            Default::default(),
+        )))?;
+        if let NetworkMessage::Headers(headers) = peer
+            .recv("headers", Some(Duration::from_secs(TIMEOUT_SECS)))?
+            .ok_or(CompactFiltersError::Timeout)?
+        {
+            let batch_len = headers.len();
+            last_hash = snapshot.apply(sync_height, headers)?;
+
+            sync_height += batch_len;
+            sync_fn(sync_height)?;
+        } else {
+            return Err(CompactFiltersError::InvalidResponse);
+        }
+    }
+
+    Ok(Some(snapshot))
+}

src/blockchain/electrum.rs

@@ -1,98 +1,127 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Electrum
+//!
+//! This module defines a [`Blockchain`] struct that wraps an [`electrum_client::Client`]
+//! and implements the logic required to populate the wallet's [database](crate::database::Database) by
+//! querying the inner client.
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::electrum::ElectrumBlockchain;
+//! let client = electrum_client::Client::new("ssl://electrum.blockstream.info:50002")?;
+//! let blockchain = ElectrumBlockchain::from(client);
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
 use std::collections::HashSet;
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
 
-use electrum_client::tokio::io::{AsyncRead, AsyncWrite};
-use electrum_client::Client;
+use electrum_client::{Client, ConfigBuilder, ElectrumApi, Socks5Config};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
 use super::*;
-use crate::database::{BatchDatabase, DatabaseUtils};
+use crate::database::BatchDatabase;
 use crate::error::Error;
+use crate::FeeRate;
 
-pub struct ElectrumBlockchain<T: AsyncRead + AsyncWrite + Send>(Option<Client<T>>);
+/// Wrapper over an Electrum Client that implements the required blockchain traits
+///
+/// ## Example
+/// See the [`blockchain::electrum`](crate::blockchain::electrum) module for a usage example.
+pub struct ElectrumBlockchain(Client);
 
-impl<T: AsyncRead + AsyncWrite + Send> std::convert::From<Client<T>> for ElectrumBlockchain<T> {
-    fn from(client: Client<T>) -> Self {
-        ElectrumBlockchain(Some(client))
+#[cfg(test)]
+#[cfg(feature = "test-electrum")]
+#[bdk_blockchain_tests(crate)]
+fn local_electrs() -> ElectrumBlockchain {
+    ElectrumBlockchain::from(Client::new(&testutils::get_electrum_url()).unwrap())
+}
+
+impl std::convert::From<Client> for ElectrumBlockchain {
+    fn from(client: Client) -> Self {
+        ElectrumBlockchain(client)
     }
 }
 
-impl<T: AsyncRead + AsyncWrite + Send> Blockchain for ElectrumBlockchain<T> {
-    fn offline() -> Self {
-        ElectrumBlockchain(None)
+impl Blockchain for ElectrumBlockchain {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        vec![
+            Capability::FullHistory,
+            Capability::GetAnyTx,
+            Capability::AccurateFees,
+        ]
+        .into_iter()
+        .collect()
     }
 
-    fn is_online(&self) -> bool {
-        self.0.is_some()
-    }
-}
-
-#[async_trait(?Send)]
-impl<T: AsyncRead + AsyncWrite + Send> OnlineBlockchain for ElectrumBlockchain<T> {
-    async fn get_capabilities(&self) -> HashSet<Capability> {
-        vec![Capability::FullHistory, Capability::GetAnyTx]
-            .into_iter()
-            .collect()
-    }
-
-    async fn setup<D: BatchDatabase + DatabaseUtils, P: Progress>(
-        &mut self,
+    fn setup<D: BatchDatabase, P: Progress>(
+        &self,
         stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
         self.0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
             .electrum_like_setup(stop_gap, database, progress_update)
-            .await
     }
 
-    async fn get_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self
-            .0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
-            .transaction_get(txid)
-            .await
-            .map(Option::Some)?)
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(self.0.transaction_get(txid).map(Option::Some)?)
     }
 
-    async fn broadcast(&mut self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self
-            .0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
-            .transaction_broadcast(tx)
-            .await
-            .map(|_| ())?)
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        Ok(self.0.transaction_broadcast(tx).map(|_| ())?)
     }
 
-    async fn get_height(&mut self) -> Result<usize, Error> {
+    fn get_height(&self) -> Result<u32, Error> {
         // TODO: unsubscribe when added to the client, or is there a better call to use here?
 
         Ok(self
             .0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
             .block_headers_subscribe()
-            .await
-            .map(|data| data.height)?)
+            .map(|data| data.height as u32)?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        Ok(FeeRate::from_btc_per_kvb(
+            self.0.estimate_fee(target)? as f32
+        ))
     }
 }
 
-#[async_trait(?Send)]
-impl<T: AsyncRead + AsyncWrite + Send> ElectrumLikeSync for Client<T> {
-    async fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
-        &mut self,
+impl ElectrumLikeSync for Client {
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
+        &self,
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
         self.batch_script_get_history(scripts)
-            .await
             .map(|v| {
                 v.into_iter()
                     .map(|v| {
@@ -112,36 +141,50 @@ impl<T: AsyncRead + AsyncWrite + Send> ElectrumLikeSync for Client<T> {
             .map_err(Error::Electrum)
     }
 
-    async fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
-        &mut self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
-        self.batch_script_list_unspent(scripts)
-            .await
-            .map(|v| {
-                v.into_iter()
-                    .map(|v| {
-                        v.into_iter()
-                            .map(
-                                |electrum_client::ListUnspentRes {
-                                     height,
-                                     tx_hash,
-                                     tx_pos,
-                                     ..
-                                 }| ELSListUnspentRes {
-                                    height,
-                                    tx_hash,
-                                    tx_pos,
-                                },
-                            )
-                            .collect()
-                    })
-                    .collect()
-            })
-            .map_err(Error::Electrum)
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid> + Clone>(
+        &self,
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
+        self.batch_transaction_get(txids).map_err(Error::Electrum)
     }
 
-    async fn els_transaction_get(&mut self, txid: &Txid) -> Result<Transaction, Error> {
-        self.transaction_get(txid).await.map_err(Error::Electrum)
+    fn els_batch_block_header<I: IntoIterator<Item = u32> + Clone>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        self.batch_block_header(heights).map_err(Error::Electrum)
+    }
+}
+
+/// Configuration for an [`ElectrumBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct ElectrumBlockchainConfig {
+    /// URL of the Electrum server (such as ElectrumX, Esplora, BWT) may start with `ssl://` or `tcp://` and include a port
+    ///
+    /// eg. `ssl://electrum.blockstream.info:60002`
+    pub url: String,
+    /// URL of the socks5 proxy server or a Tor service
+    pub socks5: Option<String>,
+    /// Request retry count
+    pub retry: u8,
+    /// Request timeout (seconds)
+    pub timeout: u8,
+}
+
+impl ConfigurableBlockchain for ElectrumBlockchain {
+    type Config = ElectrumBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        let socks5 = config.socks5.as_ref().map(Socks5Config::new);
+        let electrum_config = ConfigBuilder::new()
+            .retry(config.retry)
+            .socks5(socks5)?
+            .timeout(config.timeout)?
+            .build();
+
+        Ok(ElectrumBlockchain(Client::from_config(
+            config.url.as_str(),
+            electrum_config,
+        )?))
     }
 }

src/blockchain/esplora.rs

@@ -1,105 +1,149 @@
-use std::collections::HashSet;
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
 
-use futures::stream::{self, StreamExt, TryStreamExt};
+//! Esplora
+//!
+//! This module defines a [`Blockchain`] struct that can query an Esplora backend
+//! populate the wallet's [database](crate::database::Database) by
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use bdk::blockchain::esplora::EsploraBlockchain;
+//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/api", None);
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use std::collections::{HashMap, HashSet};
+use std::fmt;
+
+use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
 use serde::Deserialize;
 
-use reqwest::Client;
-use reqwest::StatusCode;
+use reqwest::{Client, StatusCode};
 
-use bitcoin::consensus::{deserialize, serialize};
-use bitcoin::hashes::hex::ToHex;
+use bitcoin::consensus::{self, deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
 use bitcoin::hashes::{sha256, Hash};
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHash, BlockHeader, Script, Transaction, Txid};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
 use super::*;
-use crate::database::{BatchDatabase, DatabaseUtils};
+use crate::database::BatchDatabase;
 use crate::error::Error;
+use crate::wallet::utils::ChunksIterator;
+use crate::FeeRate;
+
+const DEFAULT_CONCURRENT_REQUESTS: u8 = 4;
 
 #[derive(Debug)]
-pub struct UrlClient {
+struct UrlClient {
     url: String,
+    // We use the async client instead of the blocking one because it automatically uses `fetch`
+    // when the target platform is wasm32.
     client: Client,
+    concurrency: u8,
 }
 
+/// Structure that implements the logic to sync with Esplora
+///
+/// ## Example
+/// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example.
 #[derive(Debug)]
-pub struct EsploraBlockchain(Option<UrlClient>);
+pub struct EsploraBlockchain(UrlClient);
 
 impl std::convert::From<UrlClient> for EsploraBlockchain {
     fn from(url_client: UrlClient) -> Self {
-        EsploraBlockchain(Some(url_client))
+        EsploraBlockchain(url_client)
     }
 }
 
 impl EsploraBlockchain {
-    pub fn new(base_url: &str) -> Self {
-        EsploraBlockchain(Some(UrlClient {
+    /// Create a new instance of the client from a base URL
+    pub fn new(base_url: &str, concurrency: Option<u8>) -> Self {
+        EsploraBlockchain(UrlClient {
             url: base_url.to_string(),
             client: Client::new(),
-        }))
+            concurrency: concurrency.unwrap_or(DEFAULT_CONCURRENT_REQUESTS),
+        })
     }
 }
 
+#[maybe_async]
 impl Blockchain for EsploraBlockchain {
-    fn offline() -> Self {
-        EsploraBlockchain(None)
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        vec![
+            Capability::FullHistory,
+            Capability::GetAnyTx,
+            Capability::AccurateFees,
+        ]
+        .into_iter()
+        .collect()
     }
 
-    fn is_online(&self) -> bool {
-        self.0.is_some()
-    }
-}
-
-#[async_trait(?Send)]
-impl OnlineBlockchain for EsploraBlockchain {
-    async fn get_capabilities(&self) -> HashSet<Capability> {
-        vec![Capability::FullHistory, Capability::GetAnyTx]
-            .into_iter()
-            .collect()
-    }
-
-    async fn setup<D: BatchDatabase + DatabaseUtils, P: Progress>(
-        &mut self,
+    fn setup<D: BatchDatabase, P: Progress>(
+        &self,
         stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        self.0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
-            .electrum_like_setup(stop_gap, database, progress_update)
-            .await
+        maybe_await!(self
+            .0
+            .electrum_like_setup(stop_gap, database, progress_update))
     }
 
-    async fn get_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self
-            .0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
-            ._get_tx(txid)
-            .await?)
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(await_or_block!(self.0._get_tx(txid))?)
     }
 
-    async fn broadcast(&mut self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self
-            .0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
-            ._broadcast(tx)
-            .await?)
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        Ok(await_or_block!(self.0._broadcast(tx))?)
     }
 
-    async fn get_height(&mut self) -> Result<usize, Error> {
-        Ok(self
-            .0
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
-            ._get_height()
-            .await?)
+    fn get_height(&self) -> Result<u32, Error> {
+        Ok(await_or_block!(self.0._get_height())?)
+    }
+
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        let estimates = await_or_block!(self.0._get_fee_estimates())?;
+
+        let fee_val = estimates
+            .into_iter()
+            .map(|(k, v)| Ok::<_, std::num::ParseIntError>((k.parse::<usize>()?, v)))
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|e| Error::Generic(e.to_string()))?
+            .into_iter()
+            .take_while(|(k, _)| k <= &target)
+            .map(|(_, v)| v)
+            .last()
+            .unwrap_or(1.0);
+
+        Ok(FeeRate::from_sat_per_vb(fee_val as f32))
     }
 }
 
@@ -111,7 +155,7 @@ impl UrlClient {
     async fn _get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, EsploraError> {
         let resp = self
             .client
-            .get(&format!("{}/api/tx/{}/raw", self.url, txid))
+            .get(&format!("{}/tx/{}/raw", self.url, txid))
             .send()
             .await?;
 
@@ -122,9 +166,42 @@ impl UrlClient {
         Ok(Some(deserialize(&resp.error_for_status()?.bytes().await?)?))
     }
 
+    async fn _get_tx_no_opt(&self, txid: &Txid) -> Result<Transaction, EsploraError> {
+        match self._get_tx(txid).await {
+            Ok(Some(tx)) => Ok(tx),
+            Ok(None) => Err(EsploraError::TransactionNotFound(*txid)),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn _get_header(&self, block_height: u32) -> Result<BlockHeader, EsploraError> {
+        let resp = self
+            .client
+            .get(&format!("{}/block-height/{}", self.url, block_height))
+            .send()
+            .await?;
+
+        if let StatusCode::NOT_FOUND = resp.status() {
+            return Err(EsploraError::HeaderHeightNotFound(block_height));
+        }
+        let bytes = resp.bytes().await?;
+        let hash = std::str::from_utf8(&bytes)
+            .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?;
+
+        let resp = self
+            .client
+            .get(&format!("{}/block/{}/header", self.url, hash))
+            .send()
+            .await?;
+
+        let header = deserialize(&Vec::from_hex(&resp.text().await?)?)?;
+
+        Ok(header)
+    }
+
     async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
         self.client
-            .post(&format!("{}/api/tx", self.url))
+            .post(&format!("{}/tx", self.url))
             .body(serialize(transaction).to_hex())
             .send()
             .await?
@@ -133,16 +210,14 @@ impl UrlClient {
         Ok(())
     }
 
-    async fn _get_height(&self) -> Result<usize, EsploraError> {
-        Ok(self
+    async fn _get_height(&self) -> Result<u32, EsploraError> {
+        let req = self
             .client
-            .get(&format!("{}/api/blocks/tip/height", self.url))
+            .get(&format!("{}/blocks/tip/height", self.url))
             .send()
-            .await?
-            .error_for_status()?
-            .text()
-            .await?
-            .parse()?)
+            .await?;
+
+        Ok(req.error_for_status()?.text().await?.parse()?)
     }
 
     async fn _script_get_history(
@@ -156,7 +231,7 @@ impl UrlClient {
         result.extend(
             self.client
                 .get(&format!(
-                    "{}/api/scripthash/{}/txs/mempool",
+                    "{}/scripthash/{}/txs/mempool",
                     self.url, scripthash
                 ))
                 .send()
@@ -184,7 +259,7 @@ impl UrlClient {
             let response = self
                 .client
                 .get(&format!(
-                    "{}/api/scripthash/{}/txs/chain/{}",
+                    "{}/scripthash/{}/txs/chain/{}",
                     self.url, scripthash, last_txid
                 ))
                 .send()
@@ -212,59 +287,78 @@ impl UrlClient {
         Ok(result)
     }
 
-    async fn _script_list_unspent(
-        &self,
-        script: &Script,
-    ) -> Result<Vec<ELSListUnspentRes>, EsploraError> {
+    async fn _get_fee_estimates(&self) -> Result<HashMap<String, f64>, EsploraError> {
         Ok(self
             .client
-            .get(&format!(
-                "{}/api/scripthash/{}/utxo",
-                self.url,
-                Self::script_to_scripthash(script)
-            ))
+            .get(&format!("{}/fee-estimates", self.url,))
             .send()
             .await?
             .error_for_status()?
-            .json::<Vec<EsploraListUnspent>>()
-            .await?
-            .into_iter()
-            .map(|x| ELSListUnspentRes {
-                tx_hash: x.txid,
-                height: x.status.block_height.unwrap_or(0),
-                tx_pos: x.vout,
-            })
-            .collect())
+            .json::<HashMap<String, f64>>()
+            .await?)
     }
 }
 
-#[async_trait(?Send)]
+#[maybe_async]
 impl ElectrumLikeSync for UrlClient {
-    async fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
-        &mut self,
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
+        &self,
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
-        Ok(stream::iter(scripts)
-            .then(|script| self._script_get_history(&script))
-            .try_collect()
-            .await?)
+        let future = async {
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(scripts.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for script in chunk {
+                    futs.push(self._script_get_history(&script));
+                }
+                let partial_results: Vec<Vec<ELSGetHistoryRes>> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
+        };
+
+        await_or_block!(future)
     }
 
-    async fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
-        &mut self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
-        Ok(stream::iter(scripts)
-            .then(|script| self._script_list_unspent(&script))
-            .try_collect()
-            .await?)
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
+        &self,
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
+        let future = async {
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(txids.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for txid in chunk {
+                    futs.push(self._get_tx_no_opt(&txid));
+                }
+                let partial_results: Vec<Transaction> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
+        };
+
+        await_or_block!(future)
     }
 
-    async fn els_transaction_get(&mut self, txid: &Txid) -> Result<Transaction, Error> {
-        Ok(self
-            ._get_tx(txid)
-            .await?
-            .ok_or_else(|| EsploraError::TransactionNotFound(*txid))?)
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        let future = async {
+            let mut results = vec![];
+            for chunk in ChunksIterator::new(heights.into_iter(), self.concurrency as usize) {
+                let mut futs = FuturesOrdered::new();
+                for height in chunk {
+                    futs.push(self._get_header(height));
+                }
+                let partial_results: Vec<BlockHeader> = futs.try_collect().await?;
+                results.extend(partial_results);
+            }
+            Ok(stream::iter(results).collect().await)
+        };
+
+        await_or_block!(future)
     }
 }
 
@@ -279,36 +373,57 @@ struct EsploraGetHistory {
     status: EsploraGetHistoryStatus,
 }
 
-#[derive(Deserialize)]
-struct EsploraListUnspent {
-    txid: Txid,
-    vout: usize,
-    status: EsploraGetHistoryStatus,
+/// Configuration for an [`EsploraBlockchain`]
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct EsploraBlockchainConfig {
+    /// Base URL of the esplora service
+    ///
+    /// eg. `https://blockstream.info/api/`
+    pub base_url: String,
+    /// Number of parallel requests sent to the esplora service (default: 4)
+    pub concurrency: Option<u8>,
 }
 
+impl ConfigurableBlockchain for EsploraBlockchain {
+    type Config = EsploraBlockchainConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(EsploraBlockchain::new(
+            config.base_url.as_str(),
+            config.concurrency,
+        ))
+    }
+}
+
+/// Errors that can happen during a sync with [`EsploraBlockchain`]
 #[derive(Debug)]
 pub enum EsploraError {
+    /// Error with the HTTP call
     Reqwest(reqwest::Error),
+    /// Invalid number returned
     Parsing(std::num::ParseIntError),
+    /// Invalid Bitcoin data returned
     BitcoinEncoding(bitcoin::consensus::encode::Error),
+    /// Invalid Hex data returned
+    Hex(bitcoin::hashes::hex::Error),
 
+    /// Transaction not found
     TransactionNotFound(Txid),
+    /// Header height not found
+    HeaderHeightNotFound(u32),
+    /// Header hash not found
+    HeaderHashNotFound(BlockHash),
 }
 
-impl From<reqwest::Error> for EsploraError {
-    fn from(other: reqwest::Error) -> Self {
-        EsploraError::Reqwest(other)
+impl fmt::Display for EsploraError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
     }
 }
 
-impl From<std::num::ParseIntError> for EsploraError {
-    fn from(other: std::num::ParseIntError) -> Self {
-        EsploraError::Parsing(other)
-    }
-}
+impl std::error::Error for EsploraError {}
 
-impl From<bitcoin::consensus::encode::Error> for EsploraError {
-    fn from(other: bitcoin::consensus::encode::Error) -> Self {
-        EsploraError::BitcoinEncoding(other)
-    }
-}
+impl_error!(reqwest::Error, Reqwest, EsploraError);
+impl_error!(std::num::ParseIntError, Parsing, EsploraError);
+impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
+impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);

src/blockchain/mod.rs

@@ -1,77 +1,188 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Blockchain backends
+//!
+//! This module provides the implementation of a few commonly-used backends like
+//! [Electrum](crate::blockchain::electrum), [Esplora](crate::blockchain::esplora) and
+//! [Compact Filters/Neutrino](crate::blockchain::compact_filters), along with a generalized trait
+//! [`Blockchain`] that can be implemented to build customized backends.
+
 use std::collections::HashSet;
+use std::ops::Deref;
 use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::Arc;
 
 use bitcoin::{Transaction, Txid};
 
-use crate::database::{BatchDatabase, DatabaseUtils};
+use crate::database::BatchDatabase;
 use crate::error::Error;
+use crate::FeeRate;
 
-pub mod utils;
+#[cfg(any(feature = "electrum", feature = "esplora"))]
+pub(crate) mod utils;
+
+#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+pub mod any;
+#[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
+pub use any::{AnyBlockchain, AnyBlockchainConfig};
 
 #[cfg(feature = "electrum")]
+#[cfg_attr(docsrs, doc(cfg(feature = "electrum")))]
 pub mod electrum;
 #[cfg(feature = "electrum")]
 pub use self::electrum::ElectrumBlockchain;
+#[cfg(feature = "electrum")]
+pub use self::electrum::ElectrumBlockchainConfig;
 
 #[cfg(feature = "esplora")]
+#[cfg_attr(docsrs, doc(cfg(feature = "esplora")))]
 pub mod esplora;
 #[cfg(feature = "esplora")]
 pub use self::esplora::EsploraBlockchain;
 
+#[cfg(feature = "compact_filters")]
+#[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))]
+pub mod compact_filters;
+#[cfg(feature = "compact_filters")]
+pub use self::compact_filters::CompactFiltersBlockchain;
+
+/// Capabilities that can be supported by a [`Blockchain`] backend
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Capability {
+    /// Can recover the full history of a wallet and not only the set of currently spendable UTXOs
     FullHistory,
+    /// Can fetch any historical transaction given its txid
     GetAnyTx,
+    /// Can compute accurate fees for the transactions found during sync
+    AccurateFees,
 }
 
-pub trait Blockchain {
-    fn is_online(&self) -> bool;
+/// Marker trait for a blockchain backend
+///
+/// This is a marker trait for blockchain types. It is automatically implemented for types that
+/// implement [`Blockchain`], so as a user of the library you won't have to implement this
+/// manually.
+///
+/// Users of the library will probably never have to implement this trait manually, but they
+/// could still need to import it to define types and structs with generics;
+/// Implementing only the marker trait is pointless, since [`OfflineBlockchain`]
+/// already does that, and whenever [`Blockchain`] is implemented, the marker trait is also
+/// automatically implemented by the library.
+pub trait BlockchainMarker {}
 
-    fn offline() -> Self;
-}
+/// The [`BlockchainMarker`] marker trait is automatically implemented for [`Blockchain`] types
+impl<T: Blockchain> BlockchainMarker for T {}
 
+/// Type that only implements [`BlockchainMarker`] and is always "offline"
 pub struct OfflineBlockchain;
-impl Blockchain for OfflineBlockchain {
-    fn offline() -> Self {
-        OfflineBlockchain
-    }
+impl BlockchainMarker for OfflineBlockchain {}
 
-    fn is_online(&self) -> bool {
-        false
-    }
-}
+/// Trait that defines the actions that must be supported by a blockchain backend
+#[maybe_async]
+pub trait Blockchain: BlockchainMarker {
+    /// Return the set of [`Capability`] supported by this backend
+    fn get_capabilities(&self) -> HashSet<Capability>;
 
-#[async_trait(?Send)]
-pub trait OnlineBlockchain: Blockchain {
-    async fn get_capabilities(&self) -> HashSet<Capability>;
-
-    async fn setup<D: BatchDatabase + DatabaseUtils, P: Progress>(
-        &mut self,
+    /// Setup the backend and populate the internal database for the first time
+    ///
+    /// This method is the equivalent of [`Blockchain::sync`], but it's guaranteed to only be
+    /// called once, at the first [`Wallet::sync`](crate::wallet::Wallet::sync).
+    ///
+    /// The rationale behind the distinction between `sync` and `setup` is that some custom backends
+    /// might need to perform specific actions only the first time they are synced.
+    ///
+    /// For types that do not have that distinction, only this method can be implemented, since
+    /// [`Blockchain::sync`] defaults to calling this internally if not overridden.
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
         stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error>;
-    async fn sync<D: BatchDatabase + DatabaseUtils, P: Progress>(
-        &mut self,
+    /// Populate the internal database with transactions and UTXOs
+    ///
+    /// If not overridden, it defaults to calling [`Blockchain::setup`] internally.
+    ///
+    /// This method should implement the logic required to iterate over the list of the wallet's
+    /// script_pubkeys using [`Database::iter_script_pubkeys`] and look for relevant transactions
+    /// in the blockchain to populate the database with [`BatchOperations::set_tx`] and
+    /// [`BatchOperations::set_utxo`].
+    ///
+    /// This method should also take care of removing UTXOs that are seen as spent in the
+    /// blockchain, using [`BatchOperations::del_utxo`].
+    ///
+    /// The `progress_update` object can be used to give the caller updates about the progress by using
+    /// [`Progress::update`].
+    ///
+    /// [`Database::iter_script_pubkeys`]: crate::database::Database::iter_script_pubkeys
+    /// [`BatchOperations::set_tx`]: crate::database::BatchOperations::set_tx
+    /// [`BatchOperations::set_utxo`]: crate::database::BatchOperations::set_utxo
+    /// [`BatchOperations::del_utxo`]: crate::database::BatchOperations::del_utxo
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
         stop_gap: Option<usize>,
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        self.setup(stop_gap, database, progress_update).await
+        maybe_await!(self.setup(stop_gap, database, progress_update))
     }
 
-    async fn get_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error>;
-    async fn broadcast(&mut self, tx: &Transaction) -> Result<(), Error>;
+    /// Fetch a transaction from the blockchain given its txid
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error>;
+    /// Broadcast a transaction
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error>;
 
-    async fn get_height(&mut self) -> Result<usize, Error>;
+    /// Return the current height
+    fn get_height(&self) -> Result<u32, Error>;
+    /// Estimate the fee rate required to confirm a transaction in a given `target` of blocks
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error>;
 }
 
+/// Trait for [`Blockchain`] types that can be created given a configuration
+pub trait ConfigurableBlockchain: Blockchain + Sized {
+    /// Type that contains the configuration
+    type Config: std::fmt::Debug;
+
+    /// Create a new instance given a configuration
+    fn from_config(config: &Self::Config) -> Result<Self, Error>;
+}
+
+/// Data sent with a progress update over a [`channel`]
 pub type ProgressData = (f32, Option<String>);
 
-pub trait Progress {
+/// Trait for types that can receive and process progress updates during [`Blockchain::sync`] and
+/// [`Blockchain::setup`]
+pub trait Progress: Send {
+    /// Send a new progress update
+    ///
+    /// The `progress` value should be in the range 0.0 - 100.0, and the `message` value is an
+    /// optional text message that can be displayed to the user.
     fn update(&self, progress: f32, message: Option<String>) -> Result<(), Error>;
 }
 
+/// Shortcut to create a [`channel`] (pair of [`Sender`] and [`Receiver`]) that can transport [`ProgressData`]
 pub fn progress() -> (Sender<ProgressData>, Receiver<ProgressData>) {
     channel()
 }
@@ -87,8 +198,11 @@ impl Progress for Sender<ProgressData> {
     }
 }
 
+/// Type that implements [`Progress`] and drops every update received
+#[derive(Clone)]
 pub struct NoopProgress;
 
+/// Create a new instance of [`NoopProgress`]
 pub fn noop_progress() -> NoopProgress {
     NoopProgress
 }
@@ -98,3 +212,63 @@ impl Progress for NoopProgress {
         Ok(())
     }
 }
+
+/// Type that implements [`Progress`] and logs at level `INFO` every update received
+#[derive(Clone)]
+pub struct LogProgress;
+
+/// Create a nwe instance of [`LogProgress`]
+pub fn log_progress() -> LogProgress {
+    LogProgress
+}
+
+impl Progress for LogProgress {
+    fn update(&self, progress: f32, message: Option<String>) -> Result<(), Error> {
+        log::info!(
+            "Sync {:.3}%: `{}`",
+            progress,
+            message.unwrap_or_else(|| "".into())
+        );
+
+        Ok(())
+    }
+}
+
+#[maybe_async]
+impl<T: Blockchain> Blockchain for Arc<T> {
+    fn get_capabilities(&self) -> HashSet<Capability> {
+        maybe_await!(self.deref().get_capabilities())
+    }
+
+    fn setup<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(self.deref().setup(stop_gap, database, progress_update))
+    }
+
+    fn sync<D: BatchDatabase, P: 'static + Progress>(
+        &self,
+        stop_gap: Option<usize>,
+        database: &mut D,
+        progress_update: P,
+    ) -> Result<(), Error> {
+        maybe_await!(self.deref().sync(stop_gap, database, progress_update))
+    }
+
+    fn get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        maybe_await!(self.deref().get_tx(txid))
+    }
+    fn broadcast(&self, tx: &Transaction) -> Result<(), Error> {
+        maybe_await!(self.deref().broadcast(tx))
+    }
+
+    fn get_height(&self) -> Result<u32, Error> {
+        maybe_await!(self.deref().get_height())
+    }
+    fn estimate_fee(&self, target: usize) -> Result<FeeRate, Error> {
+        maybe_await!(self.deref().estimate_fee(target))
+    }
+}

src/blockchain/utils.rs

@@ -1,16 +1,41 @@
-use std::cmp;
-use std::collections::{HashSet, VecDeque};
-use std::convert::TryFrom;
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+use std::collections::{HashMap, HashSet};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
+use rand::seq::SliceRandom;
+use rand::thread_rng;
 
-use bitcoin::{Address, Network, OutPoint, Script, Transaction, Txid};
+use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 
 use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
-use crate::types::{ScriptType, TransactionDetails, UTXO};
+use crate::types::{KeychainKind, TransactionDetails, UTXO};
+use crate::wallet::time::Instant;
 use crate::wallet::utils::ChunksIterator;
 
 #[derive(Debug)]
@@ -19,288 +44,354 @@ pub struct ELSGetHistoryRes {
     pub tx_hash: Txid,
 }
 
-#[derive(Debug)]
-pub struct ELSListUnspentRes {
-    pub height: usize,
-    pub tx_hash: Txid,
-    pub tx_pos: usize,
-}
-
 /// Implements the synchronization logic for an Electrum-like client.
-#[async_trait(?Send)]
+#[maybe_async]
 pub trait ElectrumLikeSync {
-    async fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
-        &mut self,
+    fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script> + Clone>(
+        &self,
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error>;
 
-    async fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
-        &mut self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error>;
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid> + Clone>(
+        &self,
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error>;
 
-    async fn els_transaction_get(&mut self, txid: &Txid) -> Result<Transaction, Error>;
+    fn els_batch_block_header<I: IntoIterator<Item = u32> + Clone>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error>;
 
     // Provided methods down here...
 
-    async fn electrum_like_setup<D: BatchDatabase + DatabaseUtils, P: Progress>(
-        &mut self,
+    fn electrum_like_setup<D: BatchDatabase, P: Progress>(
+        &self,
         stop_gap: Option<usize>,
-        database: &mut D,
+        db: &mut D,
         _progress_update: P,
     ) -> Result<(), Error> {
         // TODO: progress
+        let start = Instant::new();
+        debug!("start setup");
 
         let stop_gap = stop_gap.unwrap_or(20);
-        let batch_query_size = 20;
+        let chunk_size = stop_gap;
 
-        // check unconfirmed tx, delete so they are retrieved later
-        let mut del_batch = database.begin_batch();
-        for tx in database.iter_txs(false)? {
-            if tx.height.is_none() {
-                del_batch.del_tx(&tx.txid, false)?;
-            }
-        }
-        database.commit_batch(del_batch)?;
+        let mut history_txs_id = HashSet::new();
+        let mut txid_height = HashMap::new();
+        let mut max_indexes = HashMap::new();
 
-        // maximum derivation index for a change address that we've seen during sync
-        let mut change_max_deriv = 0;
+        let mut wallet_chains = vec![KeychainKind::Internal, KeychainKind::External];
+        // shuffling improve privacy, the server doesn't know my first request is from my internal or external addresses
+        wallet_chains.shuffle(&mut thread_rng());
+        // download history of our internal and external script_pubkeys
+        for keychain in wallet_chains.iter() {
+            let script_iter = db.iter_script_pubkeys(Some(*keychain))?.into_iter();
 
-        let mut already_checked: HashSet<Script> = HashSet::new();
-        let mut to_check_later = VecDeque::with_capacity(batch_query_size);
-
-        // insert the first chunk
-        let mut iter_scriptpubkeys = database
-            .iter_script_pubkeys(Some(ScriptType::External))?
-            .into_iter();
-        let chunk: Vec<Script> = iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-        for item in chunk.into_iter().rev() {
-            to_check_later.push_front(item);
-        }
-
-        let mut iterating_external = true;
-        let mut index = 0;
-        let mut last_found = 0;
-        while !to_check_later.is_empty() {
-            trace!("to_check_later size {}", to_check_later.len());
-
-            let until = cmp::min(to_check_later.len(), batch_query_size);
-            let chunk: Vec<Script> = to_check_later.drain(..until).collect();
-            let call_result = self.els_batch_script_get_history(chunk.iter()).await?;
-
-            for (script, history) in chunk.into_iter().zip(call_result.into_iter()) {
-                trace!("received history for {:?}, size {}", script, history.len());
-
-                if !history.is_empty() {
-                    last_found = index;
-
-                    let mut check_later_scripts = self
-                        .check_history(database, script, history, &mut change_max_deriv)
-                        .await?
-                        .into_iter()
-                        .filter(|x| already_checked.insert(x.clone()))
-                        .collect();
-                    to_check_later.append(&mut check_later_scripts);
+            for (i, chunk) in ChunksIterator::new(script_iter, stop_gap).enumerate() {
+                // TODO if i == last, should create another chunk of addresses in db
+                let call_result: Vec<Vec<ELSGetHistoryRes>> =
+                    maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
+                let max_index = call_result
+                    .iter()
+                    .enumerate()
+                    .filter_map(|(i, v)| v.first().map(|_| i as u32))
+                    .max();
+                if let Some(max) = max_index {
+                    max_indexes.insert(keychain, max + (i * chunk_size) as u32);
+                }
+                let flattened: Vec<ELSGetHistoryRes> = call_result.into_iter().flatten().collect();
+                debug!("#{} of {:?} results:{}", i, keychain, flattened.len());
+                if flattened.is_empty() {
+                    // Didn't find anything in the last `stop_gap` script_pubkeys, breaking
+                    break;
                 }
 
-                index += 1;
-            }
-
-            match iterating_external {
-                true if index - last_found >= stop_gap => iterating_external = false,
-                true => {
-                    trace!("pushing one more batch from `iter_scriptpubkeys`. index = {}, last_found = {}, stop_gap = {}", index, last_found, stop_gap);
-
-                    let chunk: Vec<Script> =
-                        iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-                    for item in chunk.into_iter().rev() {
-                        to_check_later.push_front(item);
+                for el in flattened {
+                    // el.height = -1 means unconfirmed with unconfirmed parents
+                    // el.height =  0 means unconfirmed with confirmed parents
+                    // but we treat those tx the same
+                    if el.height <= 0 {
+                        txid_height.insert(el.tx_hash, None);
+                    } else {
+                        txid_height.insert(el.tx_hash, Some(el.height as u32));
                     }
-                }
-                _ => {}
-            }
-        }
-
-        // check utxo
-        // TODO: try to minimize network requests and re-use scripts if possible
-        let mut batch = database.begin_batch();
-        for chunk in ChunksIterator::new(database.iter_utxos()?.into_iter(), batch_query_size) {
-            let scripts: Vec<_> = chunk.iter().map(|u| &u.txout.script_pubkey).collect();
-            let call_result = self.els_batch_script_list_unspent(scripts).await?;
-
-            // check which utxos are actually still unspent
-            for (utxo, list_unspent) in chunk.into_iter().zip(call_result.iter()) {
-                debug!(
-                    "outpoint {:?} is unspent for me, list unspent is {:?}",
-                    utxo.outpoint, list_unspent
-                );
-
-                let mut spent = true;
-                for unspent in list_unspent {
-                    let res_outpoint = OutPoint::new(unspent.tx_hash, unspent.tx_pos as u32);
-                    if utxo.outpoint == res_outpoint {
-                        spent = false;
-                        break;
-                    }
-                }
-                if spent {
-                    info!("{} not anymore unspent, removing", utxo.outpoint);
-                    batch.del_utxo(&utxo.outpoint)?;
+                    history_txs_id.insert(el.tx_hash);
                 }
             }
         }
 
-        let current_ext = database.get_last_index(ScriptType::External)?.unwrap_or(0);
-        let first_ext_new = last_found as u32 + 1;
-        if first_ext_new > current_ext {
-            info!("Setting external index to {}", first_ext_new);
-            database.set_last_index(ScriptType::External, first_ext_new)?;
+        // saving max indexes
+        info!("max indexes are: {:?}", max_indexes);
+        for keychain in wallet_chains.iter() {
+            if let Some(index) = max_indexes.get(keychain) {
+                db.set_last_index(*keychain, *index)?;
+            }
         }
 
-        let current_int = database.get_last_index(ScriptType::Internal)?.unwrap_or(0);
-        let first_int_new = change_max_deriv + 1;
-        if first_int_new > current_int {
-            info!("Setting internal index to {}", first_int_new);
-            database.set_last_index(ScriptType::Internal, first_int_new)?;
+        // get db status
+        let txs_details_in_db: HashMap<Txid, TransactionDetails> = db
+            .iter_txs(false)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let txs_raw_in_db: HashMap<Txid, Transaction> = db
+            .iter_raw_txs()?
+            .into_iter()
+            .map(|tx| (tx.txid(), tx))
+            .collect();
+        let utxos_deps = utxos_deps(db, &txs_raw_in_db)?;
+
+        // download new txs and headers
+        let new_txs = maybe_await!(self.download_and_save_needed_raw_txs(
+            &history_txs_id,
+            &txs_raw_in_db,
+            chunk_size,
+            db
+        ))?;
+        let new_timestamps = maybe_await!(self.download_needed_headers(
+            &txid_height,
+            &txs_details_in_db,
+            chunk_size
+        ))?;
+
+        let mut batch = db.begin_batch();
+
+        // save any tx details not in db but in history_txs_id or with different height/timestamp
+        for txid in history_txs_id.iter() {
+            let height = txid_height.get(txid).cloned().flatten();
+            let timestamp = *new_timestamps.get(txid).unwrap_or(&0u64);
+            if let Some(tx_details) = txs_details_in_db.get(txid) {
+                // check if height matches, otherwise updates it
+                if tx_details.height != height {
+                    let mut new_tx_details = tx_details.clone();
+                    new_tx_details.height = height;
+                    new_tx_details.timestamp = timestamp;
+                    batch.set_tx(&new_tx_details)?;
+                }
+            } else {
+                save_transaction_details_and_utxos(
+                    &txid,
+                    db,
+                    timestamp,
+                    height,
+                    &mut batch,
+                    &utxos_deps,
+                )?;
+            }
         }
 
-        database.commit_batch(batch)?;
+        // remove any tx details in db but not in history_txs_id
+        for txid in txs_details_in_db.keys() {
+            if !history_txs_id.contains(txid) {
+                batch.del_tx(&txid, false)?;
+            }
+        }
+
+        // remove any spent utxo
+        for new_tx in new_txs.iter() {
+            for input in new_tx.input.iter() {
+                batch.del_utxo(&input.previous_output)?;
+            }
+        }
+
+        db.commit_batch(batch)?;
+        info!("finish setup, elapsed {:?}ms", start.elapsed().as_millis());
 
         Ok(())
     }
 
-    async fn check_tx_and_descendant<D: DatabaseUtils + BatchDatabase>(
-        &mut self,
-        database: &mut D,
-        txid: &Txid,
-        height: Option<u32>,
-        cur_script: &Script,
-        change_max_deriv: &mut u32,
-    ) -> Result<Vec<Script>, Error> {
-        debug!(
-            "check_tx_and_descendant of {}, height: {:?}, script: {}",
-            txid, height, cur_script
-        );
-        let mut updates = database.begin_batch();
-        let tx = match database.get_tx(&txid, true)? {
-            // TODO: do we need the raw?
-            Some(mut saved_tx) => {
-                // update the height if it's different (in case of reorg)
-                if saved_tx.height != height {
-                    info!(
-                        "updating height from {:?} to {:?} for tx {}",
-                        saved_tx.height, height, txid
-                    );
-                    saved_tx.height = height;
-                    updates.set_tx(&saved_tx)?;
-                }
-
-                debug!("already have {} in db, returning the cached version", txid);
-
-                // unwrap since we explicitly ask for the raw_tx, if it's not present something
-                // went wrong
-                saved_tx.transaction.unwrap()
-            }
-            None => self.els_transaction_get(&txid).await?,
-        };
-
-        let mut incoming: u64 = 0;
-        let mut outgoing: u64 = 0;
-
-        // look for our own inputs
-        for (i, input) in tx.input.iter().enumerate() {
-            // the fact that we visit addresses in a BFS fashion starting from the external addresses
-            // should ensure that this query is always consistent (i.e. when we get to call this all
-            // the transactions at a lower depth have already been indexed, so if an outpoint is ours
-            // we are guaranteed to have it in the db).
-            if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
-                if database.is_mine(&previous_output.script_pubkey)? {
-                    outgoing += previous_output.value;
-
-                    debug!("{} input #{} is mine, removing from utxo", txid, i);
-                    updates.del_utxo(&input.previous_output)?;
+    /// download txs identified by `history_txs_id` and theirs previous outputs if not already present in db
+    fn download_and_save_needed_raw_txs<D: BatchDatabase>(
+        &self,
+        history_txs_id: &HashSet<Txid>,
+        txs_raw_in_db: &HashMap<Txid, Transaction>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        let txids_raw_in_db: HashSet<Txid> = txs_raw_in_db.keys().cloned().collect();
+        let txids_to_download: Vec<&Txid> = history_txs_id.difference(&txids_raw_in_db).collect();
+        if !txids_to_download.is_empty() {
+            info!("got {} txs to download", txids_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                txids_to_download,
+                chunk_size,
+                db,
+            ))?);
+            let mut prev_txids = HashSet::new();
+            let mut txids_downloaded = HashSet::new();
+            for tx in txs_downloaded.iter() {
+                txids_downloaded.insert(tx.txid());
+                // add every previous input tx, but skip coinbase
+                for input in tx.input.iter().filter(|i| !i.previous_output.is_null()) {
+                    prev_txids.insert(input.previous_output.txid);
                 }
             }
+            let already_present: HashSet<Txid> =
+                txids_downloaded.union(&txids_raw_in_db).cloned().collect();
+            let prev_txs_to_download: Vec<&Txid> =
+                prev_txids.difference(&already_present).collect();
+            info!("{} previous txs to download", prev_txs_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                prev_txs_to_download,
+                chunk_size,
+                db,
+            ))?);
         }
 
-        let mut to_check_later = vec![];
-        for (i, output) in tx.output.iter().enumerate() {
-            // this output is ours, we have a path to derive it
-            if let Some((script_type, path)) =
-                database.get_path_from_script_pubkey(&output.script_pubkey)?
-            {
-                debug!("{} output #{} is mine, adding utxo", txid, i);
-                updates.set_utxo(&UTXO {
-                    outpoint: OutPoint::new(tx.txid(), i as u32),
-                    txout: output.clone(),
-                })?;
-                incoming += output.value;
-
-                if output.script_pubkey != *cur_script {
-                    debug!("{} output #{} script {} was not current script, adding script to be checked later", txid, i, output.script_pubkey);
-                    to_check_later.push(output.script_pubkey.clone())
-                }
-
-                // derive as many change addrs as external addresses that we've seen
-                if script_type == ScriptType::Internal
-                    && u32::from(path.as_ref()[0]) > *change_max_deriv
-                {
-                    *change_max_deriv = u32::from(path.as_ref()[0]);
-                }
-            }
-        }
-
-        let tx = TransactionDetails {
-            txid: tx.txid(),
-            transaction: Some(tx),
-            received: incoming,
-            sent: outgoing,
-            height,
-            timestamp: 0,
-        };
-        info!("Saving tx {}", txid);
-        updates.set_tx(&tx)?;
-
-        database.commit_batch(updates)?;
-
-        Ok(to_check_later)
+        Ok(txs_downloaded)
     }
 
-    async fn check_history<D: DatabaseUtils + BatchDatabase>(
-        &mut self,
-        database: &mut D,
-        script_pubkey: Script,
-        txs: Vec<ELSGetHistoryRes>,
-        change_max_deriv: &mut u32,
-    ) -> Result<Vec<Script>, Error> {
-        let mut to_check_later = Vec::new();
-
-        debug!(
-            "history of {} script {} has {} tx",
-            Address::from_script(&script_pubkey, Network::Testnet).unwrap(),
-            script_pubkey,
-            txs.len()
-        );
-
-        for tx in txs {
-            let height: Option<u32> = match tx.height {
-                0 | -1 => None,
-                x => u32::try_from(x).ok(),
-            };
-
-            to_check_later.extend_from_slice(
-                &self
-                    .check_tx_and_descendant(
-                        database,
-                        &tx.tx_hash,
-                        height,
-                        &script_pubkey,
-                        change_max_deriv,
-                    )
-                    .await?,
-            );
+    /// download headers at heights in `txid_height` if tx details not already present, returns a map Txid -> timestamp
+    fn download_needed_headers(
+        &self,
+        txid_height: &HashMap<Txid, Option<u32>>,
+        txs_details_in_db: &HashMap<Txid, TransactionDetails>,
+        chunk_size: usize,
+    ) -> Result<HashMap<Txid, u64>, Error> {
+        let mut txid_timestamp = HashMap::new();
+        let needed_txid_height: HashMap<&Txid, u32> = txid_height
+            .iter()
+            .filter(|(t, _)| txs_details_in_db.get(*t).is_none())
+            .filter_map(|(t, o)| o.map(|h| (t, h)))
+            .collect();
+        let needed_heights: HashSet<u32> = needed_txid_height.values().cloned().collect();
+        if !needed_heights.is_empty() {
+            info!("{} headers to download for timestamp", needed_heights.len());
+            let mut height_timestamp: HashMap<u32, u64> = HashMap::new();
+            for chunk in ChunksIterator::new(needed_heights.into_iter(), chunk_size) {
+                let call_result: Vec<BlockHeader> =
+                    maybe_await!(self.els_batch_block_header(chunk.clone()))?;
+                height_timestamp.extend(
+                    chunk
+                        .into_iter()
+                        .zip(call_result.iter().map(|h| h.time as u64)),
+                );
+            }
+            for (txid, height) in needed_txid_height {
+                let timestamp = height_timestamp
+                    .get(&height)
+                    .ok_or_else(|| Error::Generic("timestamp missing".to_string()))?;
+                txid_timestamp.insert(*txid, *timestamp);
+            }
         }
 
-        Ok(to_check_later)
+        Ok(txid_timestamp)
+    }
+
+    fn download_and_save_in_chunks<D: BatchDatabase>(
+        &self,
+        to_download: Vec<&Txid>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        for chunk in ChunksIterator::new(to_download.into_iter(), chunk_size) {
+            let call_result: Vec<Transaction> =
+                maybe_await!(self.els_batch_transaction_get(chunk))?;
+            let mut batch = db.begin_batch();
+            for new_tx in call_result.iter() {
+                batch.set_raw_tx(new_tx)?;
+            }
+            db.commit_batch(batch)?;
+            txs_downloaded.extend(call_result);
+        }
+
+        Ok(txs_downloaded)
     }
 }
+
+fn save_transaction_details_and_utxos<D: BatchDatabase>(
+    txid: &Txid,
+    db: &mut D,
+    timestamp: u64,
+    height: Option<u32>,
+    updates: &mut dyn BatchOperations,
+    utxo_deps: &HashMap<OutPoint, OutPoint>,
+) -> Result<(), Error> {
+    let tx = db.get_raw_tx(txid)?.ok_or(Error::TransactionNotFound)?;
+
+    let mut incoming: u64 = 0;
+    let mut outgoing: u64 = 0;
+
+    let mut inputs_sum: u64 = 0;
+    let mut outputs_sum: u64 = 0;
+
+    // look for our own inputs
+    for input in tx.input.iter() {
+        // skip coinbase inputs
+        if input.previous_output.is_null() {
+            continue;
+        }
+
+        // We already downloaded all previous output txs in the previous step
+        if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+            inputs_sum += previous_output.value;
+
+            if db.is_mine(&previous_output.script_pubkey)? {
+                outgoing += previous_output.value;
+            }
+        } else {
+            // The input is not ours, but we still need to count it for the fees
+            let tx = db
+                .get_raw_tx(&input.previous_output.txid)?
+                .ok_or(Error::TransactionNotFound)?;
+            inputs_sum += tx.output[input.previous_output.vout as usize].value;
+        }
+
+        // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
+        if let Some(outpoint) = utxo_deps.get(&input.previous_output) {
+            updates.del_utxo(&outpoint)?;
+        }
+    }
+
+    for (i, output) in tx.output.iter().enumerate() {
+        // to compute the fees later
+        outputs_sum += output.value;
+
+        // this output is ours, we have a path to derive it
+        if let Some((keychain, _child)) = db.get_path_from_script_pubkey(&output.script_pubkey)? {
+            debug!("{} output #{} is mine, adding utxo", txid, i);
+            updates.set_utxo(&UTXO {
+                outpoint: OutPoint::new(tx.txid(), i as u32),
+                txout: output.clone(),
+                keychain,
+            })?;
+
+            incoming += output.value;
+        }
+    }
+
+    let tx_details = TransactionDetails {
+        txid: tx.txid(),
+        transaction: Some(tx),
+        received: incoming,
+        sent: outgoing,
+        height,
+        timestamp,
+        fees: inputs_sum.saturating_sub(outputs_sum), /* if the tx is a coinbase, fees would be negative */
+    };
+    updates.set_tx(&tx_details)?;
+
+    Ok(())
+}
+
+/// returns utxo dependency as the inputs needed for the utxo to exist
+/// `tx_raw_in_db` must contains utxo's generating txs or errors witt [crate::Error::TransactionNotFound]
+fn utxos_deps<D: BatchDatabase>(
+    db: &mut D,
+    tx_raw_in_db: &HashMap<Txid, Transaction>,
+) -> Result<HashMap<OutPoint, OutPoint>, Error> {
+    let utxos = db.iter_utxos()?;
+    let mut utxos_deps = HashMap::new();
+    for utxo in utxos {
+        let from_tx = tx_raw_in_db
+            .get(&utxo.outpoint.txid)
+            .ok_or(Error::TransactionNotFound)?;
+        for input in from_tx.input.iter() {
+            utxos_deps.insert(input.previous_output, utxo.outpoint);
+        }
+    }
+    Ok(utxos_deps)
+}

src/database/any.rs

@@ -0,0 +1,388 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Runtime-checked database types
+//!
+//! This module provides the implementation of [`AnyDatabase`] which allows switching the
+//! inner [`Database`] type at runtime.
+//!
+//! ## Example
+//!
+//! In this example, `wallet_memory` and `wallet_sled` have the same type of `Wallet<OfflineBlockchain, AnyDatabase>`.
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::database::{AnyDatabase, MemoryDatabase};
+//! # use bdk::{Wallet, OfflineWallet};
+//! let memory = MemoryDatabase::default().into();
+//! let wallet_memory: OfflineWallet<AnyDatabase> =
+//!     Wallet::new_offline("...", None, Network::Testnet, memory)?;
+//!
+//! # #[cfg(feature = "key-value-db")]
+//! # {
+//! let sled = sled::open("my-database")?.open_tree("default_tree")?.into();
+//! let wallet_sled: OfflineWallet<AnyDatabase> =
+//!     Wallet::new_offline("...", None, Network::Testnet, sled)?;
+//! # }
+//! # Ok::<(), bdk::Error>(())
+//! ```
+//!
+//! When paired with the use of [`ConfigurableDatabase`], it allows creating wallets with any
+//! database supported using a single line of code:
+//!
+//! ```no_run
+//! # use bitcoin::Network;
+//! # use bdk::database::*;
+//! # use bdk::{Wallet, OfflineWallet};
+//! let config = serde_json::from_str("...")?;
+//! let database = AnyDatabase::from_config(&config)?;
+//! let wallet: OfflineWallet<_> = Wallet::new_offline("...", None, Network::Testnet, database)?;
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use super::*;
+
+macro_rules! impl_from {
+    ( $from:ty, $to:ty, $variant:ident, $( $cfg:tt )* ) => {
+        $( $cfg )*
+        impl From<$from> for $to {
+            fn from(inner: $from) -> Self {
+                <$to>::$variant(inner)
+            }
+        }
+    };
+}
+
+macro_rules! impl_inner_method {
+    ( $enum_name:ident, $self:expr, $name:ident $(, $args:expr)* ) => {
+        match $self {
+            $enum_name::Memory(inner) => inner.$name( $($args, )* ),
+            #[cfg(feature = "key-value-db")]
+            $enum_name::Sled(inner) => inner.$name( $($args, )* ),
+        }
+    }
+}
+
+/// Type that can contain any of the [`Database`] types defined by the library
+///
+/// It allows switching database type at runtime.
+///
+/// See [this module](crate::database::any)'s documentation for a usage example.
+#[derive(Debug)]
+pub enum AnyDatabase {
+    /// In-memory ephemeral database
+    Memory(memory::MemoryDatabase),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(sled::Tree),
+}
+
+impl_from!(memory::MemoryDatabase, AnyDatabase, Memory,);
+impl_from!(sled::Tree, AnyDatabase, Sled, #[cfg(feature = "key-value-db")]);
+
+/// Type that contains any of the [`BatchDatabase::Batch`] types defined by the library
+pub enum AnyBatch {
+    /// In-memory ephemeral database
+    Memory(<memory::MemoryDatabase as BatchDatabase>::Batch),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(<sled::Tree as BatchDatabase>::Batch),
+}
+
+impl_from!(
+    <memory::MemoryDatabase as BatchDatabase>::Batch,
+    AnyBatch,
+    Memory,
+);
+impl_from!(<sled::Tree as BatchDatabase>::Batch, AnyBatch, Sled, #[cfg(feature = "key-value-db")]);
+
+impl BatchOperations for AnyDatabase {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            set_script_pubkey,
+            script,
+            keychain,
+            child
+        )
+    }
+    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_utxo, utxo)
+    }
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_raw_tx, transaction)
+    }
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_tx, transaction)
+    }
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        impl_inner_method!(AnyDatabase, self, set_last_index, keychain, value)
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            del_script_pubkey_from_path,
+            keychain,
+            child
+        )
+    }
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_path_from_script_pubkey, script)
+    }
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_utxo, outpoint)
+    }
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_raw_tx, txid)
+    }
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_tx, txid, include_raw)
+    }
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyDatabase, self, del_last_index, keychain)
+    }
+}
+
+impl Database for AnyDatabase {
+    fn check_descriptor_checksum<B: AsRef<[u8]>>(
+        &mut self,
+        keychain: KeychainKind,
+        bytes: B,
+    ) -> Result<(), Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            check_descriptor_checksum,
+            keychain,
+            bytes
+        )
+    }
+
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_script_pubkeys, keychain)
+    }
+    fn iter_utxos(&self) -> Result<Vec<UTXO>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_utxos)
+    }
+    fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_raw_txs)
+    }
+    fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, iter_txs, include_raw)
+    }
+
+    fn get_script_pubkey_from_path(
+        &self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(
+            AnyDatabase,
+            self,
+            get_script_pubkey_from_path,
+            keychain,
+            child
+        )
+    }
+    fn get_path_from_script_pubkey(
+        &self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_path_from_script_pubkey, script)
+    }
+    fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_utxo, outpoint)
+    }
+    fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_raw_tx, txid)
+    }
+    fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_tx, txid, include_raw)
+    }
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyDatabase, self, get_last_index, keychain)
+    }
+
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        impl_inner_method!(AnyDatabase, self, increment_last_index, keychain)
+    }
+}
+
+impl BatchOperations for AnyBatch {
+    fn set_script_pubkey(
+        &mut self,
+        script: &Script,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_script_pubkey, script, keychain, child)
+    }
+    fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_utxo, utxo)
+    }
+    fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_raw_tx, transaction)
+    }
+    fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_tx, transaction)
+    }
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        impl_inner_method!(AnyBatch, self, set_last_index, keychain, value)
+    }
+
+    fn del_script_pubkey_from_path(
+        &mut self,
+        keychain: KeychainKind,
+        child: u32,
+    ) -> Result<Option<Script>, Error> {
+        impl_inner_method!(AnyBatch, self, del_script_pubkey_from_path, keychain, child)
+    }
+    fn del_path_from_script_pubkey(
+        &mut self,
+        script: &Script,
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
+        impl_inner_method!(AnyBatch, self, del_path_from_script_pubkey, script)
+    }
+    fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
+        impl_inner_method!(AnyBatch, self, del_utxo, outpoint)
+    }
+    fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        impl_inner_method!(AnyBatch, self, del_raw_tx, txid)
+    }
+    fn del_tx(
+        &mut self,
+        txid: &Txid,
+        include_raw: bool,
+    ) -> Result<Option<TransactionDetails>, Error> {
+        impl_inner_method!(AnyBatch, self, del_tx, txid, include_raw)
+    }
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        impl_inner_method!(AnyBatch, self, del_last_index, keychain)
+    }
+}
+
+impl BatchDatabase for AnyDatabase {
+    type Batch = AnyBatch;
+
+    fn begin_batch(&self) -> Self::Batch {
+        match self {
+            AnyDatabase::Memory(inner) => inner.begin_batch().into(),
+            #[cfg(feature = "key-value-db")]
+            AnyDatabase::Sled(inner) => inner.begin_batch().into(),
+        }
+    }
+    fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> {
+        // TODO: refactor once `move_ref_pattern` is stable
+        #[allow(irrefutable_let_patterns)]
+        match self {
+            AnyDatabase::Memory(db) => {
+                if let AnyBatch::Memory(batch) = batch {
+                    db.commit_batch(batch)
+                } else {
+                    unimplemented!()
+                }
+            }
+            #[cfg(feature = "key-value-db")]
+            AnyDatabase::Sled(db) => {
+                if let AnyBatch::Sled(batch) = batch {
+                    db.commit_batch(batch)
+                } else {
+                    unimplemented!()
+                }
+            }
+        }
+    }
+}
+
+/// Configuration type for a [`sled::Tree`] database
+#[cfg(feature = "key-value-db")]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct SledDbConfiguration {
+    /// Main directory of the db
+    pub path: String,
+    /// Name of the database tree, a separated namespace for the data
+    pub tree_name: String,
+}
+
+#[cfg(feature = "key-value-db")]
+impl ConfigurableDatabase for sled::Tree {
+    type Config = SledDbConfiguration;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(sled::open(&config.path)?.open_tree(&config.tree_name)?)
+    }
+}
+
+/// Type that can contain any of the database configurations defined by the library
+///
+/// This allows storing a single configuration that can be loaded into an [`AnyDatabase`]
+/// instance. Wallets that plan to offer users the ability to switch blockchain backend at runtime
+/// will find this particularly useful.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub enum AnyDatabaseConfig {
+    /// Memory database has no config
+    Memory(()),
+    #[cfg(feature = "key-value-db")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))]
+    /// Simple key-value embedded database based on [`sled`]
+    Sled(SledDbConfiguration),
+}
+
+impl ConfigurableDatabase for AnyDatabase {
+    type Config = AnyDatabaseConfig;
+
+    fn from_config(config: &Self::Config) -> Result<Self, Error> {
+        Ok(match config {
+            AnyDatabaseConfig::Memory(inner) => {
+                AnyDatabase::Memory(memory::MemoryDatabase::from_config(inner)?)
+            }
+            #[cfg(feature = "key-value-db")]
+            AnyDatabaseConfig::Sled(inner) => AnyDatabase::Sled(sled::Tree::from_config(inner)?),
+        })
+    }
+}
+
+impl_from!((), AnyDatabaseConfig, Memory,);
+impl_from!(SledDbConfiguration, AnyDatabaseConfig, Sled, #[cfg(feature = "key-value-db")]);

src/database/keyvalue.rs

@@ -1,10 +1,33 @@
-use std::convert::{From, TryInto};
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+use std::convert::TryInto;
 
 use sled::{Batch, Tree};
 
 use bitcoin::consensus::encode::{deserialize, serialize};
 use bitcoin::hash_types::Txid;
-use bitcoin::util::bip32::{ChildNumber, DerivationPath};
 use bitcoin::{OutPoint, Script, Transaction};
 
 use crate::database::memory::MapKey;
@@ -14,15 +37,14 @@ use crate::types::*;
 
 macro_rules! impl_batch_operations {
     ( { $($after_insert:tt)* }, $process_delete:ident ) => {
-        fn set_script_pubkey<P: AsRef<[ChildNumber]>>(&mut self, script: &Script, script_type: ScriptType, path: &P) -> Result<(), Error> {
-            let deriv_path = DerivationPath::from(path.as_ref());
-            let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key();
+        fn set_script_pubkey(&mut self, script: &Script, keychain: KeychainKind, path: u32) -> Result<(), Error> {
+            let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
             self.insert(key, serialize(script))$($after_insert)*;
 
             let key = MapKey::Script(Some(script)).as_map_key();
             let value = json!({
-                "t": script_type,
-                "p": deriv_path,
+                "t": keychain,
+                "p": path,
             });
             self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
 
@@ -31,8 +53,11 @@ macro_rules! impl_batch_operations {
 
         fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
             let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
-            let value = serialize(&utxo.txout);
-            self.insert(key, value)$($after_insert)*;
+            let value = json!({
+                "t": utxo.txout,
+                "i": utxo.keychain,
+            });
+            self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
 
             Ok(())
         }
@@ -63,23 +88,22 @@ macro_rules! impl_batch_operations {
             Ok(())
         }
 
-        fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error> {
-            let key = MapKey::LastIndex(script_type).as_map_key();
+        fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+            let key = MapKey::LastIndex(keychain).as_map_key();
             self.insert(key, &value.to_be_bytes())$($after_insert)*;
 
             Ok(())
         }
 
-        fn del_script_pubkey_from_path<P: AsRef<[ChildNumber]>>(&mut self, script_type: ScriptType, path: &P) -> Result<Option<Script>, Error> {
-            let deriv_path = DerivationPath::from(path.as_ref());
-            let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key();
+        fn del_script_pubkey_from_path(&mut self, keychain: KeychainKind, path: u32) -> Result<Option<Script>, Error> {
+            let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
             Ok(res.map_or(Ok(None), |x| Some(deserialize(&x)).transpose())?)
         }
 
-        fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result<Option<(ScriptType, DerivationPath)>, Error> {
+        fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result<Option<(KeychainKind, u32)>, Error> {
             let key = MapKey::Script(Some(script)).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
@@ -104,8 +128,11 @@ macro_rules! impl_batch_operations {
             match res {
                 None => Ok(None),
                 Some(b) => {
-                    let txout = deserialize(&b)?;
-                    Ok(Some(UTXO { outpoint: outpoint.clone(), txout }))
+                    let mut val: serde_json::Value = serde_json::from_slice(&b)?;
+                    let txout = serde_json::from_value(val["t"].take())?;
+                    let keychain = serde_json::from_value(val["i"].take())?;
+
+                    Ok(Some(UTXO { outpoint: outpoint.clone(), txout, keychain }))
                 }
             }
         }
@@ -140,8 +167,8 @@ macro_rules! impl_batch_operations {
             }
         }
 
-        fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-            let key = MapKey::LastIndex(script_type).as_map_key();
+        fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+            let key = MapKey::LastIndex(keychain).as_map_key();
             let res = self.remove(key);
             let res = $process_delete!(res);
 
@@ -179,10 +206,10 @@ impl BatchOperations for Batch {
 impl Database for Tree {
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error> {
-        let key = MapKey::DescriptorChecksum(script_type).as_map_key();
+        let key = MapKey::DescriptorChecksum(keychain).as_map_key();
 
         let prev = self.get(&key)?.map(|x| x.to_vec());
         if let Some(val) = prev {
@@ -197,8 +224,8 @@ impl Database for Tree {
         }
     }
 
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error> {
-        let key = MapKey::Path((script_type, None)).as_map_key();
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        let key = MapKey::Path((keychain, None)).as_map_key();
         self.scan_prefix(key)
             .map(|x| -> Result<_, Error> {
                 let (_, v) = x?;
@@ -213,8 +240,16 @@ impl Database for Tree {
             .map(|x| -> Result<_, Error> {
                 let (k, v) = x?;
                 let outpoint = deserialize(&k[1..])?;
-                let txout = deserialize(&v)?;
-                Ok(UTXO { outpoint, txout })
+
+                let mut val: serde_json::Value = serde_json::from_slice(&v)?;
+                let txout = serde_json::from_value(val["t"].take())?;
+                let keychain = serde_json::from_value(val["i"].take())?;
+
+                Ok(UTXO {
+                    outpoint,
+                    txout,
+                    keychain,
+                })
             })
             .collect()
     }
@@ -245,20 +280,19 @@ impl Database for Tree {
             .collect()
     }
 
-    fn get_script_pubkey_from_path<P: AsRef<[ChildNumber]>>(
+    fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
-        path: &P,
+        keychain: KeychainKind,
+        path: u32,
     ) -> Result<Option<Script>, Error> {
-        let deriv_path = DerivationPath::from(path.as_ref());
-        let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         Ok(self.get(key)?.map(|b| deserialize(&b)).transpose()?)
     }
 
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, DerivationPath)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
@@ -275,10 +309,14 @@ impl Database for Tree {
         let key = MapKey::UTXO(Some(outpoint)).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
-                let txout = deserialize(&b)?;
+                let mut val: serde_json::Value = serde_json::from_slice(&b)?;
+                let txout = serde_json::from_value(val["t"].take())?;
+                let keychain = serde_json::from_value(val["i"].take())?;
+
                 Ok(UTXO {
-                    outpoint: outpoint.clone(),
+                    outpoint: *outpoint,
                     txout,
+                    keychain,
                 })
             })
             .transpose()
@@ -303,8 +341,8 @@ impl Database for Tree {
             .transpose()
     }
 
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.get(key)?
             .map(|b| -> Result<_, Error> {
                 let array: [u8; 4] = b
@@ -318,8 +356,8 @@ impl Database for Tree {
     }
 
     // inserts 0 if not present
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.update_and_fetch(key, |prev| {
             let new = match prev {
                 Some(b) => {
@@ -358,18 +396,11 @@ impl BatchDatabase for Tree {
 
 #[cfg(test)]
 mod test {
-    use std::str::FromStr;
     use std::sync::{Arc, Condvar, Mutex, Once};
     use std::time::{SystemTime, UNIX_EPOCH};
 
     use sled::{Db, Tree};
 
-    use bitcoin::consensus::encode::deserialize;
-    use bitcoin::hashes::hex::*;
-    use bitcoin::*;
-
-    use crate::database::*;
-
     static mut COUNT: usize = 0;
 
     lazy_static! {
@@ -410,191 +441,41 @@ mod test {
 
     #[test]
     fn test_script_pubkey() {
-        let mut tree = get_tree();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        tree.set_script_pubkey(&script, script_type, &path).unwrap();
-
-        assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, &path)
-                .unwrap(),
-            Some(script.clone())
-        );
-        assert_eq!(
-            tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
-        );
+        crate::database::test::test_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_batch_script_pubkey() {
-        let mut tree = get_tree();
-        let mut batch = tree.begin_batch();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        batch
-            .set_script_pubkey(&script, script_type, &path)
-            .unwrap();
-
-        assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, &path)
-                .unwrap(),
-            None
-        );
-        assert_eq!(tree.get_path_from_script_pubkey(&script).unwrap(), None);
-
-        tree.commit_batch(batch).unwrap();
-
-        assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, &path)
-                .unwrap(),
-            Some(script.clone())
-        );
-        assert_eq!(
-            tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
-        );
+        crate::database::test::test_batch_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_iter_script_pubkey() {
-        let mut tree = get_tree();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        tree.set_script_pubkey(&script, script_type, &path).unwrap();
-
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
+        crate::database::test::test_iter_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_del_script_pubkey() {
-        let mut tree = get_tree();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        tree.set_script_pubkey(&script, script_type, &path).unwrap();
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
-
-        tree.del_script_pubkey_from_path(script_type, &path)
-            .unwrap();
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0);
+        crate::database::test::test_del_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_utxo() {
-        let mut tree = get_tree();
-
-        let outpoint = OutPoint::from_str(
-            "5df6e0e2761359d30a8275058e299fcc0381534545f55cf43e41983f5d4c9456:0",
-        )
-        .unwrap();
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let txout = TxOut {
-            value: 133742,
-            script_pubkey: script,
-        };
-        let utxo = UTXO { txout, outpoint };
-
-        tree.set_utxo(&utxo).unwrap();
-
-        assert_eq!(tree.get_utxo(&outpoint).unwrap(), Some(utxo));
+        crate::database::test::test_utxo(get_tree());
     }
 
     #[test]
     fn test_raw_tx() {
-        let mut tree = get_tree();
-
-        let hex_tx = Vec::<u8>::from_hex("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000").unwrap();
-        let tx: Transaction = deserialize(&hex_tx).unwrap();
-
-        tree.set_raw_tx(&tx).unwrap();
-
-        let txid = tx.txid();
-
-        assert_eq!(tree.get_raw_tx(&txid).unwrap(), Some(tx));
+        crate::database::test::test_raw_tx(get_tree());
     }
 
     #[test]
     fn test_tx() {
-        let mut tree = get_tree();
-
-        let hex_tx = Vec::<u8>::from_hex("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000").unwrap();
-        let tx: Transaction = deserialize(&hex_tx).unwrap();
-        let txid = tx.txid();
-        let mut tx_details = TransactionDetails {
-            transaction: Some(tx),
-            txid,
-            timestamp: 123456,
-            received: 1337,
-            sent: 420420,
-            height: Some(1000),
-        };
-
-        tree.set_tx(&tx_details).unwrap();
-
-        // get with raw tx too
-        assert_eq!(
-            tree.get_tx(&tx_details.txid, true).unwrap(),
-            Some(tx_details.clone())
-        );
-        // get only raw_tx
-        assert_eq!(
-            tree.get_raw_tx(&tx_details.txid).unwrap(),
-            tx_details.transaction
-        );
-
-        // now get without raw_tx
-        tx_details.transaction = None;
-        assert_eq!(
-            tree.get_tx(&tx_details.txid, false).unwrap(),
-            Some(tx_details)
-        );
+        crate::database::test::test_tx(get_tree());
     }
 
     #[test]
     fn test_last_index() {
-        let mut tree = get_tree();
-
-        tree.set_last_index(ScriptType::External, 1337).unwrap();
-
-        assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
-            Some(1337)
-        );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), None);
-
-        let res = tree.increment_last_index(ScriptType::External).unwrap();
-        assert_eq!(res, 1338);
-        let res = tree.increment_last_index(ScriptType::Internal).unwrap();
-        assert_eq!(res, 0);
-
-        assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
-            Some(1338)
-        );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), Some(0));
+        crate::database::test::test_last_index(get_tree());
     }
-
-    // TODO: more tests...
 }

src/database/memory.rs

@@ -1,12 +1,40 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! In-memory ephemeral database
+//!
+//! This module defines an in-memory database type called [`MemoryDatabase`] that is based on a
+//! [`BTreeMap`].
+
 use std::collections::BTreeMap;
 use std::ops::Bound::{Excluded, Included};
 
 use bitcoin::consensus::encode::{deserialize, serialize};
 use bitcoin::hash_types::Txid;
-use bitcoin::util::bip32::{ChildNumber, DerivationPath};
 use bitcoin::{OutPoint, Script, Transaction};
 
-use crate::database::{BatchDatabase, BatchOperations, Database};
+use crate::database::{BatchDatabase, BatchOperations, ConfigurableDatabase, Database};
 use crate::error::Error;
 use crate::types::*;
 
@@ -19,17 +47,17 @@ use crate::types::*;
 // descriptor checksum  d{i,e} -> vec<u8>
 
 pub(crate) enum MapKey<'a> {
-    Path((Option<ScriptType>, Option<&'a DerivationPath>)),
+    Path((Option<KeychainKind>, Option<u32>)),
     Script(Option<&'a Script>),
     UTXO(Option<&'a OutPoint>),
     RawTx(Option<&'a Txid>),
     Transaction(Option<&'a Txid>),
-    LastIndex(ScriptType),
-    DescriptorChecksum(ScriptType),
+    LastIndex(KeychainKind),
+    DescriptorChecksum(KeychainKind),
 }
 
 impl MapKey<'_> {
-    pub fn as_prefix(&self) -> Vec<u8> {
+    fn as_prefix(&self) -> Vec<u8> {
         match self {
             MapKey::Path((st, _)) => {
                 let mut v = b"p".to_vec();
@@ -49,13 +77,7 @@ impl MapKey<'_> {
 
     fn serialize_content(&self) -> Vec<u8> {
         match self {
-            MapKey::Path((_, Some(path))) => {
-                let mut res = vec![];
-                for val in *path {
-                    res.extend(&u32::from(*val).to_be_bytes());
-                }
-                res
-            }
+            MapKey::Path((_, Some(child))) => child.to_be_bytes().to_vec(),
             MapKey::Script(Some(s)) => serialize(*s),
             MapKey::UTXO(Some(s)) => serialize(*s),
             MapKey::RawTx(Some(s)) => serialize(*s),
@@ -72,24 +94,41 @@ impl MapKey<'_> {
     }
 }
 
-fn after(key: &Vec<u8>) -> Vec<u8> {
-    let mut key = key.clone();
-    let len = key.len();
-    if len > 0 {
-        // TODO i guess it could break if the value is 0xFF, but it's fine for now
-        key[len - 1] += 1;
+fn after(key: &[u8]) -> Vec<u8> {
+    let mut key = key.to_owned();
+    let mut idx = key.len();
+    while idx > 0 {
+        if key[idx - 1] == 0xFF {
+            idx -= 1;
+            continue;
+        } else {
+            key[idx - 1] += 1;
+            break;
+        }
     }
 
     key
 }
 
-#[derive(Debug)]
+/// In-memory ephemeral database
+///
+/// This database can be used as a temporary storage for wallets that are not kept permanently on
+/// a device, or on platforms that don't provide a filesystem, like `wasm32`.
+///
+/// Once it's dropped its content will be lost.
+///
+/// If you are looking for a permanent storage solution, you can try with the default key-value
+/// database called [`sled`]. See the [`database`] module documentation for more defailts.
+///
+/// [`database`]: crate::database
+#[derive(Debug, Default)]
 pub struct MemoryDatabase {
     map: BTreeMap<Vec<u8>, Box<dyn std::any::Any>>,
     deleted_keys: Vec<Vec<u8>>,
 }
 
 impl MemoryDatabase {
+    /// Create a new empty database
     pub fn new() -> Self {
         MemoryDatabase {
             map: BTreeMap::new(),
@@ -99,20 +138,19 @@ impl MemoryDatabase {
 }
 
 impl BatchOperations for MemoryDatabase {
-    fn set_script_pubkey<P: AsRef<[ChildNumber]>>(
+    fn set_script_pubkey(
         &mut self,
         script: &Script,
-        script_type: ScriptType,
-        path: &P,
+        keychain: KeychainKind,
+        path: u32,
     ) -> Result<(), Error> {
-        let deriv_path = DerivationPath::from(path.as_ref());
-        let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         self.map.insert(key, Box::new(script.clone()));
 
         let key = MapKey::Script(Some(script)).as_map_key();
         let value = json!({
-            "t": script_type,
-            "p": deriv_path,
+            "t": keychain,
+            "p": path,
         });
         self.map.insert(key, Box::new(value));
 
@@ -121,7 +159,8 @@ impl BatchOperations for MemoryDatabase {
 
     fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error> {
         let key = MapKey::UTXO(Some(&utxo.outpoint)).as_map_key();
-        self.map.insert(key, Box::new(utxo.txout.clone()));
+        self.map
+            .insert(key, Box::new((utxo.txout.clone(), utxo.keychain)));
 
         Ok(())
     }
@@ -147,20 +186,19 @@ impl BatchOperations for MemoryDatabase {
 
         Ok(())
     }
-    fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         self.map.insert(key, Box::new(value));
 
         Ok(())
     }
 
-    fn del_script_pubkey_from_path<P: AsRef<[ChildNumber]>>(
+    fn del_script_pubkey_from_path(
         &mut self,
-        script_type: ScriptType,
-        path: &P,
+        keychain: KeychainKind,
+        path: u32,
     ) -> Result<Option<Script>, Error> {
-        let deriv_path = DerivationPath::from(path.as_ref());
-        let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
@@ -169,7 +207,7 @@ impl BatchOperations for MemoryDatabase {
     fn del_path_from_script_pubkey(
         &mut self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, DerivationPath)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
@@ -193,10 +231,11 @@ impl BatchOperations for MemoryDatabase {
         match res {
             None => Ok(None),
             Some(b) => {
-                let txout = b.downcast_ref().cloned().unwrap();
+                let (txout, keychain) = b.downcast_ref().cloned().unwrap();
                 Ok(Some(UTXO {
-                    outpoint: outpoint.clone(),
+                    outpoint: *outpoint,
                     txout,
+                    keychain,
                 }))
             }
         }
@@ -233,8 +272,8 @@ impl BatchOperations for MemoryDatabase {
             }
         }
     }
-    fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         let res = self.map.remove(&key);
         self.deleted_keys.push(key);
 
@@ -248,10 +287,10 @@ impl BatchOperations for MemoryDatabase {
 impl Database for MemoryDatabase {
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error> {
-        let key = MapKey::DescriptorChecksum(script_type).as_map_key();
+        let key = MapKey::DescriptorChecksum(keychain).as_map_key();
 
         let prev = self
             .map
@@ -269,8 +308,8 @@ impl Database for MemoryDatabase {
         }
     }
 
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error> {
-        let key = MapKey::Path((script_type, None)).as_map_key();
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error> {
+        let key = MapKey::Path((keychain, None)).as_map_key();
         self.map
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(_, v)| Ok(v.downcast_ref().cloned().unwrap()))
@@ -283,8 +322,12 @@ impl Database for MemoryDatabase {
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(k, v)| {
                 let outpoint = deserialize(&k[1..]).unwrap();
-                let txout = v.downcast_ref().cloned().unwrap();
-                Ok(UTXO { outpoint, txout })
+                let (txout, keychain) = v.downcast_ref().cloned().unwrap();
+                Ok(UTXO {
+                    outpoint,
+                    txout,
+                    keychain,
+                })
             })
             .collect()
     }
@@ -313,13 +356,12 @@ impl Database for MemoryDatabase {
             .collect()
     }
 
-    fn get_script_pubkey_from_path<P: AsRef<[ChildNumber]>>(
+    fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
-        path: &P,
+        keychain: KeychainKind,
+        path: u32,
     ) -> Result<Option<Script>, Error> {
-        let deriv_path = DerivationPath::from(path.as_ref());
-        let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key();
+        let key = MapKey::Path((Some(keychain), Some(path))).as_map_key();
         Ok(self
             .map
             .get(&key)
@@ -329,7 +371,7 @@ impl Database for MemoryDatabase {
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, DerivationPath)>, Error> {
+    ) -> Result<Option<(KeychainKind, u32)>, Error> {
         let key = MapKey::Script(Some(script)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
             let mut val: serde_json::Value = b.downcast_ref().cloned().unwrap();
@@ -343,10 +385,11 @@ impl Database for MemoryDatabase {
     fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error> {
         let key = MapKey::UTXO(Some(outpoint)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
-            let txout = b.downcast_ref().cloned().unwrap();
+            let (txout, keychain) = b.downcast_ref().cloned().unwrap();
             UTXO {
-                outpoint: outpoint.clone(),
+                outpoint: *outpoint,
                 txout,
+                keychain,
             }
         }))
     }
@@ -371,19 +414,19 @@ impl Database for MemoryDatabase {
         }))
     }
 
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         Ok(self.map.get(&key).map(|b| *b.downcast_ref().unwrap()))
     }
 
     // inserts 0 if not present
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error> {
-        let key = MapKey::LastIndex(script_type).as_map_key();
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
+        let key = MapKey::LastIndex(keychain).as_map_key();
         let value = self
             .map
-            .entry(key.clone())
+            .entry(key)
             .and_modify(|x| *x.downcast_mut::<u32>().unwrap() += 1)
-            .or_insert(Box::<u32>::new(0))
+            .or_insert_with(|| Box::<u32>::new(0))
             .downcast_mut()
             .unwrap();
 
@@ -402,23 +445,80 @@ impl BatchDatabase for MemoryDatabase {
         for key in batch.deleted_keys {
             self.map.remove(&key);
         }
+        self.map.append(&mut batch.map);
+        Ok(())
+    }
+}
 
-        Ok(self.map.append(&mut batch.map))
+impl ConfigurableDatabase for MemoryDatabase {
+    type Config = ();
+
+    fn from_config(_config: &Self::Config) -> Result<Self, Error> {
+        Ok(MemoryDatabase::default())
+    }
+}
+
+#[cfg(test)]
+impl MemoryDatabase {
+    // Artificially insert a tx in the database, as if we had found it with a `sync`
+    pub fn received_tx(
+        &mut self,
+        tx_meta: testutils::TestIncomingTx,
+        current_height: Option<u32>,
+    ) -> bitcoin::Txid {
+        use std::str::FromStr;
+
+        let tx = Transaction {
+            version: 1,
+            lock_time: 0,
+            input: vec![],
+            output: tx_meta
+                .output
+                .iter()
+                .map(|out_meta| bitcoin::TxOut {
+                    value: out_meta.value,
+                    script_pubkey: bitcoin::Address::from_str(&out_meta.to_address)
+                        .unwrap()
+                        .script_pubkey(),
+                })
+                .collect(),
+        };
+
+        let txid = tx.txid();
+        let height = tx_meta
+            .min_confirmations
+            .map(|conf| current_height.unwrap().checked_sub(conf as u32).unwrap());
+
+        let tx_details = TransactionDetails {
+            transaction: Some(tx.clone()),
+            txid,
+            timestamp: 0,
+            height,
+            received: 0,
+            sent: 0,
+            fees: 0,
+        };
+
+        self.set_tx(&tx_details).unwrap();
+        for (vout, out) in tx.output.iter().enumerate() {
+            self.set_utxo(&UTXO {
+                txout: out.clone(),
+                outpoint: OutPoint {
+                    txid,
+                    vout: vout as u32,
+                },
+                keychain: KeychainKind::External,
+            })
+            .unwrap();
+        }
+
+        txid
     }
 }
 
 #[cfg(test)]
 mod test {
-    use std::str::FromStr;
-    use std::sync::{Arc, Condvar, Mutex, Once};
-    use std::time::{SystemTime, UNIX_EPOCH};
-
-    use bitcoin::consensus::encode::deserialize;
-    use bitcoin::hashes::hex::*;
-    use bitcoin::*;
-
-    use super::*;
-    use crate::database::*;
+    use super::MemoryDatabase;
 
     fn get_tree() -> MemoryDatabase {
         MemoryDatabase::new()
@@ -426,215 +526,41 @@ mod test {
 
     #[test]
     fn test_script_pubkey() {
-        let mut tree = get_tree();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        tree.set_script_pubkey(&script, script_type, &path).unwrap();
-
-        assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, &path)
-                .unwrap(),
-            Some(script.clone())
-        );
-        assert_eq!(
-            tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
-        );
+        crate::database::test::test_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_batch_script_pubkey() {
-        let mut tree = get_tree();
-        let mut batch = tree.begin_batch();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        batch
-            .set_script_pubkey(&script, script_type, &path)
-            .unwrap();
-
-        assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, &path)
-                .unwrap(),
-            None
-        );
-        assert_eq!(tree.get_path_from_script_pubkey(&script).unwrap(), None);
-
-        tree.commit_batch(batch).unwrap();
-
-        assert_eq!(
-            tree.get_script_pubkey_from_path(script_type, &path)
-                .unwrap(),
-            Some(script.clone())
-        );
-        assert_eq!(
-            tree.get_path_from_script_pubkey(&script).unwrap(),
-            Some((script_type, path.clone()))
-        );
+        crate::database::test::test_batch_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_iter_script_pubkey() {
-        let mut tree = get_tree();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        tree.set_script_pubkey(&script, script_type, &path).unwrap();
-
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
+        crate::database::test::test_iter_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_del_script_pubkey() {
-        let mut tree = get_tree();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        tree.set_script_pubkey(&script, script_type, &path).unwrap();
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
-
-        tree.del_script_pubkey_from_path(script_type, &path)
-            .unwrap();
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0);
-    }
-
-    #[test]
-    fn test_del_script_pubkey_batch() {
-        let mut tree = get_tree();
-
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let path = DerivationPath::from_str("m/0/1/2/3").unwrap();
-        let script_type = ScriptType::External;
-
-        tree.set_script_pubkey(&script, script_type, &path).unwrap();
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
-
-        let mut batch = tree.begin_batch();
-        batch
-            .del_script_pubkey_from_path(script_type, &path)
-            .unwrap();
-
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
-
-        tree.commit_batch(batch);
-        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0);
+        crate::database::test::test_del_script_pubkey(get_tree());
     }
 
     #[test]
     fn test_utxo() {
-        let mut tree = get_tree();
-
-        let outpoint = OutPoint::from_str(
-            "5df6e0e2761359d30a8275058e299fcc0381534545f55cf43e41983f5d4c9456:0",
-        )
-        .unwrap();
-        let script = Script::from(
-            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
-        );
-        let txout = TxOut {
-            value: 133742,
-            script_pubkey: script,
-        };
-        let utxo = UTXO { txout, outpoint };
-
-        tree.set_utxo(&utxo).unwrap();
-
-        assert_eq!(tree.get_utxo(&outpoint).unwrap(), Some(utxo));
+        crate::database::test::test_utxo(get_tree());
     }
 
     #[test]
     fn test_raw_tx() {
-        let mut tree = get_tree();
-
-        let hex_tx = Vec::<u8>::from_hex("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000").unwrap();
-        let tx: Transaction = deserialize(&hex_tx).unwrap();
-
-        tree.set_raw_tx(&tx).unwrap();
-
-        let txid = tx.txid();
-
-        assert_eq!(tree.get_raw_tx(&txid).unwrap(), Some(tx));
+        crate::database::test::test_raw_tx(get_tree());
     }
 
     #[test]
     fn test_tx() {
-        let mut tree = get_tree();
-
-        let hex_tx = Vec::<u8>::from_hex("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000").unwrap();
-        let tx: Transaction = deserialize(&hex_tx).unwrap();
-        let txid = tx.txid();
-        let mut tx_details = TransactionDetails {
-            transaction: Some(tx),
-            txid,
-            timestamp: 123456,
-            received: 1337,
-            sent: 420420,
-            height: Some(1000),
-        };
-
-        tree.set_tx(&tx_details).unwrap();
-
-        // get with raw tx too
-        assert_eq!(
-            tree.get_tx(&tx_details.txid, true).unwrap(),
-            Some(tx_details.clone())
-        );
-        // get only raw_tx
-        assert_eq!(
-            tree.get_raw_tx(&tx_details.txid).unwrap(),
-            tx_details.transaction
-        );
-
-        // now get without raw_tx
-        tx_details.transaction = None;
-        assert_eq!(
-            tree.get_tx(&tx_details.txid, false).unwrap(),
-            Some(tx_details)
-        );
+        crate::database::test::test_tx(get_tree());
     }
 
     #[test]
     fn test_last_index() {
-        let mut tree = get_tree();
-
-        tree.set_last_index(ScriptType::External, 1337).unwrap();
-
-        assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
-            Some(1337)
-        );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), None);
-
-        let res = tree.increment_last_index(ScriptType::External).unwrap();
-        assert_eq!(res, 1338);
-        let res = tree.increment_last_index(ScriptType::Internal).unwrap();
-        assert_eq!(res, 0);
-
-        assert_eq!(
-            tree.get_last_index(ScriptType::External).unwrap(),
-            Some(1338)
-        );
-        assert_eq!(tree.get_last_index(ScriptType::Internal).unwrap(), Some(0));
+        crate::database::test::test_last_index(get_tree());
     }
-
-    // TODO: more tests...
 }

src/database/mod.rs

@@ -1,83 +1,177 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Database types
+//!
+//! This module provides the implementation of some defaults database types, along with traits that
+//! can be implemented externally to let [`Wallet`]s use customized databases.
+//!
+//! It's important to note that the databases defined here only contains "blockchain-related" data.
+//! They can be seen more as a cache than a critical piece of storage that contains secrets and
+//! keys.
+//!
+//! The currently recommended database is [`sled`], which is a pretty simple key-value embedded
+//! database written in Rust. If the `key-value-db` feature is enabled (which by default is),
+//! this library automatically implements all the required traits for [`sled::Tree`].
+//!
+//! [`Wallet`]: crate::wallet::Wallet
+
 use bitcoin::hash_types::Txid;
-use bitcoin::util::bip32::{ChildNumber, DerivationPath};
 use bitcoin::{OutPoint, Script, Transaction, TxOut};
 
 use crate::error::Error;
 use crate::types::*;
 
-#[cfg(any(feature = "key-value-db", feature = "default"))]
-pub mod keyvalue;
-pub mod memory;
+pub mod any;
+pub use any::{AnyDatabase, AnyDatabaseConfig};
 
+#[cfg(feature = "key-value-db")]
+pub(crate) mod keyvalue;
+
+pub mod memory;
+pub use memory::MemoryDatabase;
+
+/// Trait for operations that can be batched
+///
+/// This trait defines the list of operations that must be implemented on the [`Database`] type and
+/// the [`BatchDatabase::Batch`] type.
 pub trait BatchOperations {
-    fn set_script_pubkey<P: AsRef<[ChildNumber]>>(
+    /// Store a script_pubkey along with its keychain and child number.
+    fn set_script_pubkey(
         &mut self,
         script: &Script,
-        script_type: ScriptType,
-        path: &P,
+        keychain: KeychainKind,
+        child: u32,
     ) -> Result<(), Error>;
+    /// Store a [`UTXO`]
     fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error>;
+    /// Store a raw transaction
     fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error>;
+    /// Store the metadata of a transaction
     fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error>;
-    fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error>;
+    /// Store the last derivation index for a given keychain.
+    fn set_last_index(&mut self, keychain: KeychainKind, value: u32) -> Result<(), Error>;
 
-    fn del_script_pubkey_from_path<P: AsRef<[ChildNumber]>>(
+    /// Delete a script_pubkey given the keychain and its child number.
+    fn del_script_pubkey_from_path(
         &mut self,
-        script_type: ScriptType,
-        path: &P,
+        keychain: KeychainKind,
+        child: u32,
     ) -> Result<Option<Script>, Error>;
+    /// Delete the data related to a specific script_pubkey, meaning the keychain and the child
+    /// number.
     fn del_path_from_script_pubkey(
         &mut self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, DerivationPath)>, Error>;
+    ) -> Result<Option<(KeychainKind, u32)>, Error>;
+    /// Delete a [`UTXO`] given its [`OutPoint`]
     fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
+    /// Delete a raw transaction given its [`Txid`]
     fn del_raw_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error>;
+    /// Delete the metadata of a transaction and optionally the raw transaction itself
     fn del_tx(
         &mut self,
         txid: &Txid,
         include_raw: bool,
     ) -> Result<Option<TransactionDetails>, Error>;
-    fn del_last_index(&mut self, script_type: ScriptType) -> Result<Option<u32>, Error>;
+    /// Delete the last derivation index for a keychain.
+    fn del_last_index(&mut self, keychain: KeychainKind) -> Result<Option<u32>, Error>;
 }
 
+/// Trait for reading data from a database
+///
+/// This traits defines the operations that can be used to read data out of a database
 pub trait Database: BatchOperations {
+    /// Read and checks the descriptor checksum for a given keychain.
+    ///
+    /// Should return [`Error::ChecksumMismatch`](crate::error::Error::ChecksumMismatch) if the
+    /// checksum doesn't match. If there's no checksum in the database, simply store it for the
+    /// next time.
     fn check_descriptor_checksum<B: AsRef<[u8]>>(
         &mut self,
-        script_type: ScriptType,
+        keychain: KeychainKind,
         bytes: B,
     ) -> Result<(), Error>;
 
-    fn iter_script_pubkeys(&self, script_type: Option<ScriptType>) -> Result<Vec<Script>, Error>;
+    /// Return the list of script_pubkeys
+    fn iter_script_pubkeys(&self, keychain: Option<KeychainKind>) -> Result<Vec<Script>, Error>;
+    /// Return the list of [`UTXO`]s
     fn iter_utxos(&self) -> Result<Vec<UTXO>, Error>;
+    /// Return the list of raw transactions
     fn iter_raw_txs(&self) -> Result<Vec<Transaction>, Error>;
+    /// Return the list of transactions metadata
     fn iter_txs(&self, include_raw: bool) -> Result<Vec<TransactionDetails>, Error>;
 
-    fn get_script_pubkey_from_path<P: AsRef<[ChildNumber]>>(
+    /// Fetch a script_pubkey given the child number of a keychain.
+    fn get_script_pubkey_from_path(
         &self,
-        script_type: ScriptType,
-        path: &P,
+        keychain: KeychainKind,
+        child: u32,
     ) -> Result<Option<Script>, Error>;
+    /// Fetch the keychain and child number of a given script_pubkey
     fn get_path_from_script_pubkey(
         &self,
         script: &Script,
-    ) -> Result<Option<(ScriptType, DerivationPath)>, Error>;
+    ) -> Result<Option<(KeychainKind, u32)>, Error>;
+    /// Fetch a [`UTXO`] given its [`OutPoint`]
     fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<UTXO>, Error>;
+    /// Fetch a raw transaction given its [`Txid`]
     fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error>;
+    /// Fetch the transaction metadata and optionally also the raw transaction
     fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result<Option<TransactionDetails>, Error>;
-    fn get_last_index(&self, script_type: ScriptType) -> Result<Option<u32>, Error>;
+    /// Return the last defivation index for a keychain.
+    fn get_last_index(&self, keychain: KeychainKind) -> Result<Option<u32>, Error>;
 
-    // inserts 0 if not present
-    fn increment_last_index(&mut self, script_type: ScriptType) -> Result<u32, Error>;
+    /// Increment the last derivation index for a keychain and return it
+    ///
+    /// It should insert and return `0` if not present in the database
+    fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error>;
 }
 
+/// Trait for a database that supports batch operations
+///
+/// This trait defines the methods to start and apply a batch of operations.
 pub trait BatchDatabase: Database {
+    /// Container for the operations
     type Batch: BatchOperations;
 
+    /// Create a new batch container
     fn begin_batch(&self) -> Self::Batch;
+    /// Consume and apply a batch of operations
     fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error>;
 }
 
-pub trait DatabaseUtils: Database {
+/// Trait for [`Database`] types that can be created given a configuration
+pub trait ConfigurableDatabase: Database + Sized {
+    /// Type that contains the configuration
+    type Config: std::fmt::Debug;
+
+    /// Create a new instance given a configuration
+    fn from_config(config: &Self::Config) -> Result<Self, Error>;
+}
+
+pub(crate) trait DatabaseUtils: Database {
     fn is_mine(&self, script: &Script) -> Result<bool, Error> {
         self.get_path_from_script_pubkey(script)
             .map(|o| o.is_some())
@@ -95,11 +189,11 @@ pub trait DatabaseUtils: Database {
 
     fn get_previous_output(&self, outpoint: &OutPoint) -> Result<Option<TxOut>, Error> {
         self.get_raw_tx(&outpoint.txid)?
-            .and_then(|previous_tx| {
+            .map(|previous_tx| {
                 if outpoint.vout as usize >= previous_tx.output.len() {
-                    Some(Err(Error::InvalidOutpoint(outpoint.clone())))
+                    Err(Error::InvalidOutpoint(*outpoint))
                 } else {
-                    Some(Ok(previous_tx.output[outpoint.vout as usize].clone()))
+                    Ok(previous_tx.output[outpoint.vout as usize].clone())
                 }
             })
             .transpose()
@@ -107,3 +201,183 @@ pub trait DatabaseUtils: Database {
 }
 
 impl<T: Database> DatabaseUtils for T {}
+
+#[cfg(test)]
+pub mod test {
+    use std::str::FromStr;
+
+    use bitcoin::consensus::encode::deserialize;
+    use bitcoin::hashes::hex::*;
+    use bitcoin::*;
+
+    use super::*;
+
+    pub fn test_script_pubkey<D: Database>(mut tree: D) {
+        let script = Script::from(
+            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
+        );
+        let path = 42;
+        let keychain = KeychainKind::External;
+
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
+
+        assert_eq!(
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
+            Some(script.clone())
+        );
+        assert_eq!(
+            tree.get_path_from_script_pubkey(&script).unwrap(),
+            Some((keychain, path.clone()))
+        );
+    }
+
+    pub fn test_batch_script_pubkey<D: BatchDatabase>(mut tree: D) {
+        let mut batch = tree.begin_batch();
+
+        let script = Script::from(
+            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
+        );
+        let path = 42;
+        let keychain = KeychainKind::External;
+
+        batch.set_script_pubkey(&script, keychain, path).unwrap();
+
+        assert_eq!(
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
+            None
+        );
+        assert_eq!(tree.get_path_from_script_pubkey(&script).unwrap(), None);
+
+        tree.commit_batch(batch).unwrap();
+
+        assert_eq!(
+            tree.get_script_pubkey_from_path(keychain, path).unwrap(),
+            Some(script.clone())
+        );
+        assert_eq!(
+            tree.get_path_from_script_pubkey(&script).unwrap(),
+            Some((keychain, path.clone()))
+        );
+    }
+
+    pub fn test_iter_script_pubkey<D: Database>(mut tree: D) {
+        let script = Script::from(
+            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
+        );
+        let path = 42;
+        let keychain = KeychainKind::External;
+
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
+
+        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
+    }
+
+    pub fn test_del_script_pubkey<D: Database>(mut tree: D) {
+        let script = Script::from(
+            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
+        );
+        let path = 42;
+        let keychain = KeychainKind::External;
+
+        tree.set_script_pubkey(&script, keychain, path).unwrap();
+        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1);
+
+        tree.del_script_pubkey_from_path(keychain, path).unwrap();
+        assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0);
+    }
+
+    pub fn test_utxo<D: Database>(mut tree: D) {
+        let outpoint = OutPoint::from_str(
+            "5df6e0e2761359d30a8275058e299fcc0381534545f55cf43e41983f5d4c9456:0",
+        )
+        .unwrap();
+        let script = Script::from(
+            Vec::<u8>::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(),
+        );
+        let txout = TxOut {
+            value: 133742,
+            script_pubkey: script,
+        };
+        let utxo = UTXO {
+            txout,
+            outpoint,
+            keychain: KeychainKind::External,
+        };
+
+        tree.set_utxo(&utxo).unwrap();
+
+        assert_eq!(tree.get_utxo(&outpoint).unwrap(), Some(utxo));
+    }
+
+    pub fn test_raw_tx<D: Database>(mut tree: D) {
+        let hex_tx = Vec::<u8>::from_hex("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000").unwrap();
+        let tx: Transaction = deserialize(&hex_tx).unwrap();
+
+        tree.set_raw_tx(&tx).unwrap();
+
+        let txid = tx.txid();
+
+        assert_eq!(tree.get_raw_tx(&txid).unwrap(), Some(tx));
+    }
+
+    pub fn test_tx<D: Database>(mut tree: D) {
+        let hex_tx = Vec::<u8>::from_hex("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000").unwrap();
+        let tx: Transaction = deserialize(&hex_tx).unwrap();
+        let txid = tx.txid();
+        let mut tx_details = TransactionDetails {
+            transaction: Some(tx),
+            txid,
+            timestamp: 123456,
+            received: 1337,
+            sent: 420420,
+            fees: 140,
+            height: Some(1000),
+        };
+
+        tree.set_tx(&tx_details).unwrap();
+
+        // get with raw tx too
+        assert_eq!(
+            tree.get_tx(&tx_details.txid, true).unwrap(),
+            Some(tx_details.clone())
+        );
+        // get only raw_tx
+        assert_eq!(
+            tree.get_raw_tx(&tx_details.txid).unwrap(),
+            tx_details.transaction
+        );
+
+        // now get without raw_tx
+        tx_details.transaction = None;
+        assert_eq!(
+            tree.get_tx(&tx_details.txid, false).unwrap(),
+            Some(tx_details)
+        );
+    }
+
+    pub fn test_last_index<D: Database>(mut tree: D) {
+        tree.set_last_index(KeychainKind::External, 1337).unwrap();
+
+        assert_eq!(
+            tree.get_last_index(KeychainKind::External).unwrap(),
+            Some(1337)
+        );
+        assert_eq!(tree.get_last_index(KeychainKind::Internal).unwrap(), None);
+
+        let res = tree.increment_last_index(KeychainKind::External).unwrap();
+        assert_eq!(res, 1338);
+        let res = tree.increment_last_index(KeychainKind::Internal).unwrap();
+        assert_eq!(res, 0);
+
+        assert_eq!(
+            tree.get_last_index(KeychainKind::External).unwrap(),
+            Some(1338)
+        );
+        assert_eq!(
+            tree.get_last_index(KeychainKind::Internal).unwrap(),
+            Some(0)
+        );
+    }
+
+    // TODO: more tests...
+}

src/descriptor/checksum.rs

@@ -1,3 +1,32 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Descriptor checksum
+//!
+//! This module contains a re-implementation of the function used by Bitcoin Core to calculate the
+//! checksum of a descriptor
+
 use std::iter::FromIterator;
 
 use crate::descriptor::Error;
@@ -27,6 +56,7 @@ fn poly_mod(mut c: u64, val: u64) -> u64 {
     c
 }
 
+/// Compute the checksum of a descriptor
 pub fn get_checksum(desc: &str) -> Result<String, Error> {
     let mut c = 1;
     let mut cls = 0;
@@ -62,3 +92,35 @@ pub fn get_checksum(desc: &str) -> Result<String, Error> {
 
     Ok(String::from_iter(chars))
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::descriptor::get_checksum;
+
+    // test get_checksum() function; it should return the same value as Bitcoin Core
+    #[test]
+    fn test_get_checksum() {
+        let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)";
+        assert_eq!(get_checksum(desc).unwrap(), "tqz0nc62");
+
+        let desc = "pkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/44'/1'/0'/0/*)";
+        assert_eq!(get_checksum(desc).unwrap(), "lasegmfs");
+    }
+
+    #[test]
+    fn test_get_checksum_invalid_character() {
+        let sparkle_heart = vec![240, 159, 146, 150];
+        let sparkle_heart = std::str::from_utf8(&sparkle_heart)
+            .unwrap()
+            .chars()
+            .next()
+            .unwrap();
+        let invalid_desc = format!("wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcL{}fjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)", sparkle_heart);
+
+        assert!(matches!(
+            get_checksum(&invalid_desc).err(),
+            Some(Error::InvalidDescriptorCharacter(invalid_char)) if invalid_char == sparkle_heart
+        ));
+    }
+}

src/descriptor/dsl.rs

@@ -0,0 +1,981 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Descriptors DSL
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_top_level_sh {
+    // disallow `sortedmulti` in `bare()`
+    ( Bare, Bare, sortedmulti $( $inner:tt )* ) => {
+        compile_error!("`bare()` descriptors can't contain any `sortedmulti()` operands");
+    };
+    ( Bare, Bare, sortedmulti_vec $( $inner:tt )* ) => {
+        compile_error!("`bare()` descriptors can't contain any `sortedmulti_vec()` operands");
+    };
+
+    ( $descriptor_variant:ident, $sortedmulti_variant:ident, sortedmulti $( $inner:tt )* ) => {
+        $crate::impl_sortedmulti!(sortedmulti $( $inner )*)
+            .and_then(|(inner, key_map, valid_networks)| Ok(($crate::miniscript::Descriptor::$sortedmulti_variant(inner), key_map, valid_networks)))
+    };
+    ( $descriptor_variant:ident, $sortedmulti_variant:ident, sortedmulti_vec $( $inner:tt )* ) => {
+        $crate::impl_sortedmulti!(sortedmulti_vec $( $inner )*)
+            .and_then(|(inner, key_map, valid_networks)| Ok(($crate::miniscript::Descriptor::$sortedmulti_variant(inner), key_map, valid_networks)))
+    };
+
+    ( $descriptor_variant:ident, $sortedmulti_variant:ident, $( $minisc:tt )* ) => {
+        $crate::fragment!($( $minisc )*)
+            .map(|(minisc, keymap, networks)|($crate::miniscript::Descriptor::<$crate::miniscript::descriptor::DescriptorPublicKey>::$descriptor_variant(minisc), keymap, networks))
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_top_level_pk {
+    ( $descriptor_variant:ident, $ctx:ty, $key:expr ) => {{
+        #[allow(unused_imports)]
+        use $crate::keys::{DescriptorKey, ToDescriptorKey};
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+
+        $key.to_descriptor_key()
+            .and_then(|key: DescriptorKey<$ctx>| key.extract(&secp))
+            .map(|(pk, key_map, valid_networks)| {
+                (
+                    $crate::miniscript::Descriptor::<
+                        $crate::miniscript::descriptor::DescriptorPublicKey,
+                    >::$descriptor_variant(pk),
+                    key_map,
+                    valid_networks,
+                )
+            })
+    }};
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_leaf_opcode {
+    ( $terminal_variant:ident ) => {
+        $crate::miniscript::Miniscript::from_ast(
+            $crate::miniscript::miniscript::decode::Terminal::$terminal_variant,
+        )
+        .map_err($crate::Error::Miniscript)
+        .map(|minisc| {
+            (
+                minisc,
+                $crate::miniscript::descriptor::KeyMap::default(),
+                $crate::keys::any_network(),
+            )
+        })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_leaf_opcode_value {
+    ( $terminal_variant:ident, $value:expr ) => {
+        $crate::miniscript::Miniscript::from_ast(
+            $crate::miniscript::miniscript::decode::Terminal::$terminal_variant($value),
+        )
+        .map_err($crate::Error::Miniscript)
+        .map(|minisc| {
+            (
+                minisc,
+                $crate::miniscript::descriptor::KeyMap::default(),
+                $crate::keys::any_network(),
+            )
+        })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_leaf_opcode_value_two {
+    ( $terminal_variant:ident, $one:expr, $two:expr ) => {
+        $crate::miniscript::Miniscript::from_ast(
+            $crate::miniscript::miniscript::decode::Terminal::$terminal_variant($one, $two),
+        )
+        .map_err($crate::Error::Miniscript)
+        .map(|minisc| {
+            (
+                minisc,
+                $crate::miniscript::descriptor::KeyMap::default(),
+                $crate::keys::any_network(),
+            )
+        })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_node_opcode_two {
+    ( $terminal_variant:ident, $( $inner:tt )* ) => ({
+        let inner = $crate::fragment_internal!( @t $( $inner )* );
+        let (a, b) = $crate::descriptor::dsl::TupleTwo::from(inner).flattened();
+
+        a
+            .and_then(|a| Ok((a, b?)))
+            .and_then(|((a_minisc, mut a_keymap, a_networks), (b_minisc, b_keymap, b_networks))| {
+                // join key_maps
+                a_keymap.extend(b_keymap.into_iter());
+
+                Ok(($crate::miniscript::Miniscript::from_ast($crate::miniscript::miniscript::decode::Terminal::$terminal_variant(
+                    std::sync::Arc::new(a_minisc),
+                    std::sync::Arc::new(b_minisc),
+                ))?, a_keymap, $crate::keys::merge_networks(&a_networks, &b_networks)))
+            })
+    });
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_node_opcode_three {
+    ( $terminal_variant:ident, $( $inner:tt )* ) => {
+        let inner = $crate::fragment_internal!( @t $( $inner )* );
+        let (a, b, c) = $crate::descriptor::dsl::TupleThree::from(inner).flattened();
+
+        a
+            .and_then(|a| Ok((a, b?, c?)))
+            .and_then(|((a_minisc, mut a_keymap, a_networks), (b_minisc, b_keymap, b_networks), (c_minisc, c_keymap, c_networks))| {
+                // join key_maps
+                a_keymap.extend(b_keymap.into_iter());
+                a_keymap.extend(c_keymap.into_iter());
+
+                let networks = $crate::keys::merge_networks(&a_networks, &b_networks);
+                let networks = $crate::keys::merge_networks(&networks, &c_networks);
+
+                Ok(($crate::miniscript::Miniscript::from_ast($crate::miniscript::miniscript::decode::Terminal::$terminal_variant(
+                    std::sync::Arc::new(a_minisc),
+                    std::sync::Arc::new(b_minisc),
+                    std::sync::Arc::new(c_minisc),
+                ))?, a_keymap, networks))
+            })
+    };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! impl_sortedmulti {
+    ( sortedmulti_vec ( $thresh:expr, $keys:expr ) ) => ({
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+        $crate::keys::make_sortedmulti_inner($thresh, $keys, &secp)
+    });
+    ( sortedmulti ( $thresh:expr $(, $key:expr )+ ) ) => ({
+        use $crate::keys::ToDescriptorKey;
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+
+        let mut keys = vec![];
+        $(
+            keys.push($key.to_descriptor_key());
+        )*
+
+        keys.into_iter().collect::<Result<Vec<_>, _>>()
+            .and_then(|keys| $crate::keys::make_sortedmulti_inner($thresh, keys, &secp))
+    });
+
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! apply_modifier {
+    ( $terminal_variant:ident, $inner:expr ) => {{
+        $inner
+            .map_err(|e| -> $crate::Error { e.into() })
+            .and_then(|(minisc, keymap, networks)| {
+                Ok((
+                    $crate::miniscript::Miniscript::from_ast(
+                        $crate::miniscript::miniscript::decode::Terminal::$terminal_variant(
+                            std::sync::Arc::new(minisc),
+                        ),
+                    )?,
+                    keymap,
+                    networks,
+                ))
+            })
+    }};
+
+    ( a: $inner:expr ) => {{
+        $crate::apply_modifier!(Alt, $inner)
+    }};
+    ( s: $inner:expr ) => {{
+        $crate::apply_modifier!(Swap, $inner)
+    }};
+    ( c: $inner:expr ) => {{
+        $crate::apply_modifier!(Check, $inner)
+    }};
+    ( d: $inner:expr ) => {{
+        $crate::apply_modifier!(DupIf, $inner)
+    }};
+    ( v: $inner:expr ) => {{
+        $crate::apply_modifier!(Verify, $inner)
+    }};
+    ( j: $inner:expr ) => {{
+        $crate::apply_modifier!(NonZero, $inner)
+    }};
+    ( n: $inner:expr ) => {{
+        $crate::apply_modifier!(ZeroNotEqual, $inner)
+    }};
+
+    // Modifiers expanded to other operators
+    ( t: $inner:expr ) => {{
+        $inner.and_then(|(a_minisc, a_keymap, a_networks)| {
+            $crate::impl_leaf_opcode_value_two!(
+                AndV,
+                std::sync::Arc::new(a_minisc),
+                std::sync::Arc::new($crate::fragment!(true).unwrap().0)
+            )
+            .map(|(minisc, _, _)| (minisc, a_keymap, a_networks))
+        })
+    }};
+    ( l: $inner:expr ) => {{
+        $inner.and_then(|(a_minisc, a_keymap, a_networks)| {
+            $crate::impl_leaf_opcode_value_two!(
+                OrI,
+                std::sync::Arc::new($crate::fragment!(false).unwrap().0),
+                std::sync::Arc::new(a_minisc)
+            )
+            .map(|(minisc, _, _)| (minisc, a_keymap, a_networks))
+        })
+    }};
+    ( u: $inner:expr ) => {{
+        $inner.and_then(|(a_minisc, a_keymap, a_networks)| {
+            $crate::impl_leaf_opcode_value_two!(
+                OrI,
+                std::sync::Arc::new(a_minisc),
+                std::sync::Arc::new($crate::fragment!(false).unwrap().0)
+            )
+            .map(|(minisc, _, _)| (minisc, a_keymap, a_networks))
+        })
+    }};
+}
+
+/// Macro to write full descriptors with code
+///
+/// This macro expands to a `Result` of
+/// [`DescriptorTemplateOut`](super::template::DescriptorTemplateOut) and [`Error`](crate::Error)
+///
+/// The syntax is very similar to the normal descriptor syntax, with the exception that modifiers
+/// cannot be grouped together. For instance, a descriptor fragment like `sdv:older(144)` has to be
+/// broken up to `s:d:v:older(144)`.
+///
+/// The `pk()`, `pk_k()` and `pk_h()` operands can take as argument any type that implements
+/// [`ToDescriptorKey`]. This means that keys can also be written inline as strings, but in that
+/// case they must be wrapped in quotes, which is another difference compared to the standard
+/// descriptor syntax.
+///
+/// [`ToDescriptorKey`]: crate::keys::ToDescriptorKey
+///
+/// ## Example
+///
+/// Signature plus timelock descriptor:
+///
+/// ```
+/// # use std::str::FromStr;
+/// let (my_descriptor, my_keys_map, networks) = bdk::descriptor!(sh(wsh(and_v(v:pk("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy"),older(50)))))?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+///
+/// -------
+///
+/// 2-of-3 that becomes a 1-of-3 after a timelock has expired. Both `descriptor_a` and `descriptor_b` are equivalent: the first
+/// syntax is more suitable for a fixed number of items known at compile time, while the other accepts a
+/// [`Vec`] of items, which makes it more suitable for writing dynamic descriptors.
+///
+/// They both produce the descriptor: `wsh(thresh(2,pk(...),s:pk(...),sdv:older(...)))`
+///
+/// ```
+/// # use std::str::FromStr;
+/// let my_key_1 = bitcoin::PublicKey::from_str("02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c")?;
+/// let my_key_2 = bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
+/// let my_timelock = 50;
+///
+/// let (descriptor_a, key_map_a, networks) = bdk::descriptor! {
+///     wsh (
+///         thresh(2, pk(my_key_1), s:pk(my_key_2), s:d:v:older(my_timelock))
+///     )
+/// }?;
+///
+/// let b_items = vec![
+///     bdk::fragment!(pk(my_key_1))?,
+///     bdk::fragment!(s:pk(my_key_2))?,
+///     bdk::fragment!(s:d:v:older(my_timelock))?,
+/// ];
+/// let (descriptor_b, mut key_map_b, networks) = bdk::descriptor!(wsh(thresh_vec(2,b_items)))?;
+///
+/// assert_eq!(descriptor_a, descriptor_b);
+/// assert_eq!(key_map_a.len(), key_map_b.len());
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+///
+/// ------
+///
+/// Simple 2-of-2 multi-signature, equivalent to: `wsh(multi(2, ...))`
+///
+/// ```
+/// # use std::str::FromStr;
+/// let my_key_1 = bitcoin::PublicKey::from_str(
+///     "02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c",
+/// )?;
+/// let my_key_2 =
+///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
+///
+/// let (descriptor, key_map, networks) = bdk::descriptor! {
+///     wsh (
+///         multi(2, my_key_1, my_key_2)
+///     )
+/// }?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+///
+/// ------
+///
+/// Native-Segwit single-sig, equivalent to: `wpkh(...)`
+///
+/// ```
+/// let my_key =
+///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
+///
+/// let (descriptor, key_map, networks) = bdk::descriptor!(wpkh(my_key))?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+#[macro_export]
+macro_rules! descriptor {
+    ( bare ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(Bare, Bare, $( $minisc )*)
+    });
+    ( sh ( wsh ( $( $minisc:tt )* ) ) ) => ({
+        $crate::descriptor!(shwsh ($( $minisc )*))
+    });
+    ( shwsh ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(ShWsh, ShWshSortedMulti, $( $minisc )*)
+    });
+    ( pk ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(Pk, $crate::miniscript::Legacy, $key)
+    });
+    ( pkh ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(Pkh,$crate::miniscript::Legacy, $key)
+    });
+    ( wpkh ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(Wpkh, $crate::miniscript::Segwitv0, $key)
+    });
+    ( sh ( wpkh ( $key:expr ) ) ) => ({
+        $crate::descriptor!(shwpkh ( $key ))
+    });
+    ( shwpkh ( $key:expr ) ) => ({
+        $crate::impl_top_level_pk!(ShWpkh, $crate::miniscript::Segwitv0, $key)
+    });
+    ( sh ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(Sh, ShSortedMulti, $( $minisc )*)
+    });
+    ( wsh ( $( $minisc:tt )* ) ) => ({
+        $crate::impl_top_level_sh!(Wsh, WshSortedMulti, $( $minisc )*)
+    });
+}
+
+#[doc(hidden)]
+pub struct TupleTwo<A, B> {
+    pub a: A,
+    pub b: B,
+}
+
+impl<A, B> TupleTwo<A, B> {
+    pub fn flattened(self) -> (A, B) {
+        (self.a, self.b)
+    }
+}
+
+impl<A, B> From<(A, (B, ()))> for TupleTwo<A, B> {
+    fn from((a, (b, _)): (A, (B, ()))) -> Self {
+        TupleTwo { a, b }
+    }
+}
+
+#[doc(hidden)]
+pub struct TupleThree<A, B, C> {
+    pub a: A,
+    pub b: B,
+    pub c: C,
+}
+
+impl<A, B, C> TupleThree<A, B, C> {
+    pub fn flattened(self) -> (A, B, C) {
+        (self.a, self.b, self.c)
+    }
+}
+
+impl<A, B, C> From<(A, (B, (C, ())))> for TupleThree<A, B, C> {
+    fn from((a, (b, (c, _))): (A, (B, (C, ())))) -> Self {
+        TupleThree { a, b, c }
+    }
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! fragment_internal {
+    // The @v prefix is used to parse a sequence of operands and return them in a vector. This is
+    // used by operands that take a variable number of arguments, like `thresh()` and `multi()`.
+    ( @v $op:ident ( $( $args:tt )* ) $( $tail:tt )* ) => ({
+        let mut v = vec![$crate::fragment!( $op ( $( $args )* ) )];
+        v.append(&mut $crate::fragment_internal!( @v $( $tail )* ));
+
+        v
+    });
+    // Match modifiers
+    ( @v $modif:tt : $( $tail:tt )* ) => ({
+        let mut v = $crate::fragment_internal!( @v $( $tail )* );
+        let first = v.drain(..1).next().unwrap();
+
+        let first = $crate::apply_modifier!($modif:first);
+
+        let mut v_final = vec![first];
+        v_final.append(&mut v);
+
+        v_final
+    });
+    // Remove commas between operands
+    ( @v , $( $tail:tt )* ) => ({
+        $crate::fragment_internal!( @v $( $tail )* )
+    });
+    ( @v ) => ({
+        vec![]
+    });
+
+    // The @t prefix is used to parse a sequence of operands and return them in a tuple. This
+    // allows checking at compile-time the number of arguments passed to an operand. For this
+    // reason it's used by `and_*()`, `or_*()`, etc.
+    //
+    // Unfortunately, due to the fact that concatenating tuples is pretty hard, the final result
+    // adds in the first spot the parsed operand and in the second spot the result of parsing
+    // all the following ones. For two operands the type then corresponds to: (X, (X, ())). For
+    // three operands it's (X, (X, (X, ()))), etc.
+    //
+    // To check that the right number of arguments has been passed we can "cast" those tuples to
+    // more convenient structures like `TupleTwo`. If the conversion succedes, the right number of
+    // args was passed. Otherwise the compilation fails entirely.
+    ( @t $op:ident ( $( $args:tt )* ) $( $tail:tt )* ) => ({
+        ($crate::fragment!( $op ( $( $args )* ) ), $crate::fragment_internal!( @t $( $tail )* ))
+    });
+    // Match modifiers
+    ( @t $modif:tt : $( $tail:tt )* ) => ({
+        let (first, tail) = $crate::fragment_internal!( @t $( $tail )* );
+        ($crate::apply_modifier!($modif:first), tail)
+    });
+    // Remove commas between operands
+    ( @t , $( $tail:tt )* ) => ({
+        $crate::fragment_internal!( @t $( $tail )* )
+    });
+    ( @t ) => ({
+        ()
+    });
+
+    // Fallback to calling `fragment!()`
+    ( $( $tokens:tt )* ) => ({
+        $crate::fragment!($( $tokens )*)
+    });
+}
+
+/// Macro to write descriptor fragments with code
+///
+/// This macro will be expanded to an object of type `Result<(Miniscript<DescriptorPublicKey, _>, KeyMap, ValidNetworks), Error>`. It allows writing
+/// fragments of larger descriptors that can be pieced together using `fragment!(thresh_vec(m, ...))`.
+///
+/// The syntax to write macro fragment is the same as documented for the [`descriptor`] macro.
+#[macro_export]
+macro_rules! fragment {
+    // Modifiers
+    ( $modif:tt : $( $tail:tt )* ) => ({
+        let op = $crate::fragment!( $( $tail )* );
+        $crate::apply_modifier!($modif:op)
+    });
+
+    // Miniscript
+    ( true ) => ({
+        $crate::impl_leaf_opcode!(True)
+    });
+    ( false ) => ({
+        $crate::impl_leaf_opcode!(False)
+    });
+    ( pk_k ( $key:expr ) ) => ({
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+        $crate::keys::make_pk($key, &secp)
+    });
+    ( pk ( $key:expr ) ) => ({
+        $crate::fragment!(c:pk_k ( $key ))
+    });
+    ( pk_h ( $key_hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(PkH, $key_hash)
+    });
+    ( after ( $value:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(After, $value)
+    });
+    ( older ( $value:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Older, $value)
+    });
+    ( sha256 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Sha256, $hash)
+    });
+    ( hash256 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Hash256, $hash)
+    });
+    ( ripemd160 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Ripemd160, $hash)
+    });
+    ( hash160 ( $hash:expr ) ) => ({
+        $crate::impl_leaf_opcode_value!(Hash160, $hash)
+    });
+    ( and_v ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(AndV, $( $inner )*)
+    });
+    ( and_b ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(AndB, $( $inner )*)
+    });
+    ( and_or ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_three!(AndOr, $( $inner )*)
+    });
+    ( or_b ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrB, $( $inner )*)
+    });
+    ( or_d ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrD, $( $inner )*)
+    });
+    ( or_c ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrC, $( $inner )*)
+    });
+    ( or_i ( $( $inner:tt )* ) ) => ({
+        $crate::impl_node_opcode_two!(OrI, $( $inner )*)
+    });
+    ( thresh_vec ( $thresh:expr, $items:expr ) ) => ({
+        use $crate::miniscript::descriptor::KeyMap;
+
+        let (items, key_maps_networks): (Vec<_>, Vec<_>) = $items.into_iter().map(|(a, b, c)| (a, (b, c))).unzip();
+        let items = items.into_iter().map(std::sync::Arc::new).collect();
+
+        let (key_maps, valid_networks) = key_maps_networks.into_iter().fold((KeyMap::default(), $crate::keys::any_network()), |(mut keys_acc, net_acc), (key, net)| {
+            keys_acc.extend(key.into_iter());
+            let net_acc = $crate::keys::merge_networks(&net_acc, &net);
+
+            (keys_acc, net_acc)
+        });
+
+        $crate::impl_leaf_opcode_value_two!(Thresh, $thresh, items)
+            .map(|(minisc, _, _)| (minisc, key_maps, valid_networks))
+    });
+    ( thresh ( $thresh:expr, $( $inner:tt )* ) ) => ({
+        let items = $crate::fragment_internal!( @v $( $inner )* );
+
+        items.into_iter().collect::<Result<Vec<_>, _>>()
+            .and_then(|items| $crate::fragment!(thresh_vec($thresh, items)))
+    });
+    ( multi_vec ( $thresh:expr, $keys:expr ) ) => ({
+        $crate::keys::make_multi($thresh, $keys)
+    });
+    ( multi ( $thresh:expr $(, $key:expr )+ ) ) => ({
+        use $crate::keys::ToDescriptorKey;
+        let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
+
+        let mut keys = vec![];
+        $(
+            keys.push($key.to_descriptor_key());
+        )*
+
+        keys.into_iter().collect::<Result<Vec<_>, _>>()
+            .and_then(|keys| $crate::keys::make_multi($thresh, keys, &secp))
+    });
+
+    // `sortedmulti()` is handled separately
+    ( sortedmulti ( $( $inner:tt )* ) ) => ({
+        compile_error!("`sortedmulti` can only be used as the root operand of a descriptor");
+    });
+    ( sortedmulti_vec ( $( $inner:tt )* ) ) => ({
+        compile_error!("`sortedmulti_vec` can only be used as the root operand of a descriptor");
+    });
+}
+
+#[cfg(test)]
+mod test {
+    use bitcoin::hashes::hex::ToHex;
+    use bitcoin::secp256k1::Secp256k1;
+    use miniscript::descriptor::{DescriptorPublicKey, DescriptorPublicKeyCtx, KeyMap};
+    use miniscript::{Descriptor, Legacy, Segwitv0};
+
+    use std::str::FromStr;
+
+    use crate::descriptor::DescriptorMeta;
+    use crate::keys::{DescriptorKey, KeyError, ToDescriptorKey, ValidNetworks};
+    use bitcoin::network::constants::Network::{Bitcoin, Regtest, Testnet};
+    use bitcoin::util::bip32;
+    use bitcoin::util::bip32::ChildNumber;
+    use bitcoin::PrivateKey;
+
+    // test the descriptor!() macro
+
+    // verify descriptor generates expected script(s) (if bare or pk) or address(es)
+    fn check(
+        desc: Result<(Descriptor<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError>,
+        is_witness: bool,
+        is_fixed: bool,
+        expected: &[&str],
+    ) {
+        let secp = Secp256k1::new();
+        let deriv_ctx = DescriptorPublicKeyCtx::new(&secp, ChildNumber::Normal { index: 0 });
+
+        let (desc, _key_map, _networks) = desc.unwrap();
+        assert_eq!(desc.is_witness(), is_witness);
+        assert_eq!(desc.is_fixed(), is_fixed);
+        for i in 0..expected.len() {
+            let index = i as u32;
+            let child_desc = if desc.is_fixed() {
+                desc.clone()
+            } else {
+                desc.derive(ChildNumber::from_normal_idx(index).unwrap())
+            };
+            let address = child_desc.address(Regtest, deriv_ctx);
+            if let Some(address) = address {
+                assert_eq!(address.to_string(), *expected.get(i).unwrap());
+            } else {
+                let script = child_desc.script_pubkey(deriv_ctx);
+                assert_eq!(script.to_hex().as_str(), *expected.get(i).unwrap());
+            }
+        }
+    }
+
+    // - at least one of each "type" of operator; ie. one modifier, one leaf_opcode, one leaf_opcode_value, etc.
+    // - mixing up key types that implement ToDescriptorKey in multi() or thresh()
+
+    // expected script for pk and bare manually created
+    // expected addresses created with `bitcoin-cli getdescriptorinfo` (for hash) and `bitcoin-cli deriveaddresses`
+
+    #[test]
+    fn test_fixed_legacy_descriptors() {
+        let pubkey1 = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        let pubkey2 = bitcoin::PublicKey::from_str(
+            "032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af",
+        )
+        .unwrap();
+
+        check(
+            descriptor!(bare(multi(1,pubkey1,pubkey2))),
+            false,
+            true,
+            &["512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd21032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af52ae"],
+        );
+        check(
+            descriptor!(pk(pubkey1)),
+            false,
+            true,
+            &["2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac"],
+        );
+        check(
+            descriptor!(pkh(pubkey1)),
+            false,
+            true,
+            &["muZpTpBYhxmRFuCjLc7C6BBDF32C8XVJUi"],
+        );
+        check(
+            descriptor!(sh(multi(1, pubkey1, pubkey2))),
+            false,
+            true,
+            &["2MymURoV1bzuMnWMGiXzyomDkeuxXY7Suey"],
+        );
+    }
+
+    #[test]
+    fn test_fixed_segwitv0_descriptors() {
+        let pubkey1 = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        let pubkey2 = bitcoin::PublicKey::from_str(
+            "032e58afe51f9ed8ad3cc7897f634d881fdbe49a81564629ded8156bebd2ffd1af",
+        )
+        .unwrap();
+
+        check(
+            descriptor!(wpkh(pubkey1)),
+            true,
+            true,
+            &["bcrt1qngw83fg8dz0k749cg7k3emc7v98wy0c7azaa6h"],
+        );
+        check(
+            descriptor!(sh(wpkh(pubkey1))),
+            true,
+            true,
+            &["2N5LiC3CqzxDamRTPG1kiNv1FpNJQ7x28sb"],
+        );
+        check(
+            descriptor!(wsh(multi(1, pubkey1, pubkey2))),
+            true,
+            true,
+            &["bcrt1qgw8jvv2hsrvjfa6q66rk6har7d32lrqm5unnf5cl63q9phxfvgps5fyfqe"],
+        );
+        check(
+            descriptor!(sh(wsh(multi(1, pubkey1, pubkey2)))),
+            true,
+            true,
+            &["2NCidRJysy7apkmE6JF5mLLaJFkrN3Ub9iy"],
+        );
+    }
+
+    #[test]
+    fn test_bip32_legacy_descriptors() {
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(pk(desc_key)),
+            false,
+            false,
+            &[
+                "2102363ad03c10024e1b597a5b01b9982807fb638e00b06f3b2d4a89707de3b93c37ac",
+                "2102063a21fd780df370ed2fc8c4b86aa5ea642630609c203009df631feb7b480dd2ac",
+                "2102ba2685ad1fa5891cb100f1656b2ce3801822ccb9bac0336734a6f8c1b93ebbc0ac",
+            ],
+        );
+
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(pkh(desc_key)),
+            false,
+            false,
+            &[
+                "muvBdsVpJxpFuTHMKA47htJPdCvdt4F9DP",
+                "mxQSHK7DL2t1DN3xFxov1janCoXSSkrSPj",
+                "mfz43r15GiWo4nizmyzMNubsnkDpByFFAn",
+            ],
+        );
+
+        let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
+        let desc_key1 = (xprv, path).to_descriptor_key().unwrap();
+        let desc_key2 = (xprv, path2).to_descriptor_key().unwrap();
+
+        check(
+            descriptor!(sh(multi(1, desc_key1, desc_key2))),
+            false,
+            false,
+            &[
+                "2MtMDXsfwefZkEEhVViEPidvcKRUtJamJJ8",
+                "2MwAUZ1NYyWjhVvGTethFL6n7nZhS8WE6At",
+                "2MuT6Bj66HLwZd7s4SoD8XbK4GwriKEA6Gr",
+            ],
+        );
+    }
+
+    #[test]
+    fn test_bip32_segwitv0_descriptors() {
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(wpkh(desc_key)),
+            true,
+            false,
+            &[
+                "bcrt1qnhm8w9fhc8cxzgqsmqdf9fyjccyvc0gltnymu0",
+                "bcrt1qhylfd55rn75w9fj06zspctad5w4hz33rf0ttad",
+                "bcrt1qq5sq3a6k9av9d8cne0k9wcldy4nqey5yt6889r",
+            ],
+        );
+
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(sh(wpkh(desc_key))),
+            true,
+            false,
+            &[
+                "2MxvjQCaLqZ5QxZ7XotZDQ63hZw3NPss763",
+                "2NDUoevN4QMzhvHDMGhKuiT2fN9HXbFRMwn",
+                "2NF4BEAY2jF1Fu8vqfN3NVKoFtom77pUxrx",
+            ],
+        );
+
+        let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
+        let desc_key1 = (xprv, path.clone()).to_descriptor_key().unwrap();
+        let desc_key2 = (xprv, path2.clone()).to_descriptor_key().unwrap();
+        check(
+            descriptor!(wsh(multi(1, desc_key1, desc_key2))),
+            true,
+            false,
+            &[
+                "bcrt1qfxv8mxmlv5sz8q2mnuyaqdfe9jr4vvmx0csjhn092p6f4qfygfkq2hng49",
+                "bcrt1qerj85g243e6jlcdxpmn9spk0gefcwvu7nw7ee059d5ydzpdhkm2qwfkf5k",
+                "bcrt1qxkl2qss3k58q9ktc8e89pwr4gnptfpw4hju4xstxcjc0hkcae3jstluty7",
+            ],
+        );
+
+        let desc_key1 = (xprv, path).to_descriptor_key().unwrap();
+        let desc_key2 = (xprv, path2).to_descriptor_key().unwrap();
+        check(
+            descriptor!(sh(wsh(multi(1, desc_key1, desc_key2)))),
+            true,
+            false,
+            &[
+                "2NFCtXvx9q4ci2kvKub17iSTgvRXGctCGhz",
+                "2NB2PrFPv5NxWCpygas8tPrGJG2ZFgeuwJw",
+                "2N79ZAGo5cMi5Jt7Wo9L5YmF5GkEw7sjWdC",
+            ],
+        );
+    }
+
+    #[test]
+    fn test_dsl_sortedmulti() {
+        let key_1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path_1 = bip32::DerivationPath::from_str("m/0").unwrap();
+
+        let key_2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let path_2 = bip32::DerivationPath::from_str("m/1").unwrap();
+
+        let desc_key1 = (key_1, path_1);
+        let desc_key2 = (key_2, path_2);
+
+        check(
+            descriptor!(sh(sortedmulti(1, desc_key1.clone(), desc_key2.clone()))),
+            false,
+            false,
+            &[
+                "2MsxzPEJDBzpGffJXPaDpfXZAUNnZhaMh2N",
+                "2My3x3DLPK3UbGWGpxrXr1RnbD8MNC4FpgS",
+                "2NByEuiQT7YLqHCTNxL5KwYjvtuCYcXNBSC",
+                "2N1TGbP81kj2VUKTSWgrwxoMfuWjvfUdyu7",
+                "2N3Bomq2fpAcLRNfZnD3bCWK9quan28CxCR",
+                "2N9nrZaEzEFDqEAU9RPvDnXGT6AVwBDKAQb",
+            ],
+        );
+
+        check(
+            descriptor!(sh(wsh(sortedmulti(
+                1,
+                desc_key1.clone(),
+                desc_key2.clone()
+            )))),
+            true,
+            false,
+            &[
+                "2NCogc5YyM4N6ruv1hUa7WLMW1BPeCK7N9B",
+                "2N6mkSAKi1V2oaBXby7XHdvBMKEDRQcFpNe",
+                "2NFmTSttm9v6bXeoWaBvpMcgfPQcZhNn3Eh",
+                "2Mvib87RBPUHXNEpX5S5Kv1qqrhBfgBGsJM",
+                "2MtMv5mcK2EjcLsH8Txpx2JxLLzHr4ttczL",
+                "2MsWCB56rb4T6yPv8QudZGHERTwNgesE4f6",
+            ],
+        );
+
+        check(
+            descriptor!(wsh(sortedmulti_vec(1, vec![desc_key1, desc_key2]))),
+            true,
+            false,
+            &[
+                "bcrt1qcvq0lg8q7a47ytrd7zk5y7uls7mulrenjgvflwylpppgwf8029es4vhpnj",
+                "bcrt1q80yn8sdt6l7pjvkz25lglyaqctlmsq9ugk80rmxt8yu0npdsj97sc7l4de",
+                "bcrt1qrvf6024v9s50qhffe3t2fr2q9ckdhx2g6jz32chm2pp24ymgtr5qfrdmct",
+                "bcrt1q6srfmra0ynypym35c7jvsxt2u4yrugeajq95kg2ps7lk6h2gaunsq9lzxn",
+                "bcrt1qhl8rrzzcdpu7tcup3lcg7tge52sqvwy5fcv4k78v6kxtwmqf3v6qpvyjza",
+                "bcrt1ql2elz9mhm9ll27ddpewhxs732xyl2fk2kpkqz9gdyh33wgcun4vstrd49k",
+            ],
+        );
+    }
+
+    // - verify the valid_networks returned is correctly computed based on the keys present in the descriptor
+    #[test]
+    fn test_valid_networks() {
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+
+        let (_desc, _key_map, valid_networks) = descriptor!(pkh(desc_key)).unwrap();
+        assert_eq!(valid_networks, [Testnet, Regtest].iter().cloned().collect());
+
+        let xprv = bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi").unwrap();
+        let path = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
+        let desc_key = (xprv, path.clone()).to_descriptor_key().unwrap();
+
+        let (_desc, _key_map, valid_networks) = descriptor!(wpkh(desc_key)).unwrap();
+        assert_eq!(valid_networks, [Bitcoin].iter().cloned().collect());
+    }
+
+    // - verify the key_maps are correctly merged together
+    #[test]
+    fn test_key_maps_merged() {
+        let secp = Secp256k1::new();
+
+        let xprv1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path1 = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key1 = (xprv1, path1.clone()).to_descriptor_key().unwrap();
+
+        let xprv2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
+        let desc_key2 = (xprv2, path2.clone()).to_descriptor_key().unwrap();
+
+        let xprv3 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf").unwrap();
+        let path3 = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
+        let desc_key3 = (xprv3, path3.clone()).to_descriptor_key().unwrap();
+
+        let (_desc, key_map, _valid_networks) =
+            descriptor!(sh(wsh(multi(2, desc_key1, desc_key2, desc_key3)))).unwrap();
+        assert_eq!(key_map.len(), 3);
+
+        let desc_key1: DescriptorKey<Segwitv0> =
+            (xprv1, path1.clone()).to_descriptor_key().unwrap();
+        let desc_key2: DescriptorKey<Segwitv0> =
+            (xprv2, path2.clone()).to_descriptor_key().unwrap();
+        let desc_key3: DescriptorKey<Segwitv0> =
+            (xprv3, path3.clone()).to_descriptor_key().unwrap();
+
+        let (key1, _key_map, _valid_networks) = desc_key1.extract(&secp).unwrap();
+        let (key2, _key_map, _valid_networks) = desc_key2.extract(&secp).unwrap();
+        let (key3, _key_map, _valid_networks) = desc_key3.extract(&secp).unwrap();
+        assert_eq!(key_map.get(&key1).unwrap().to_string(), "tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy/0/*");
+        assert_eq!(key_map.get(&key2).unwrap().to_string(), "tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF/2147483647'/0/*");
+        assert_eq!(key_map.get(&key3).unwrap().to_string(), "tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf/10/20/30/40/*");
+    }
+
+    // - verify the ScriptContext is correctly validated (i.e. passing a type that only impl ToDescriptorKey<Segwitv0> to a pkh() descriptor should throw a compilation error
+    #[test]
+    fn test_script_context_validation() {
+        // this compiles
+        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let path = bip32::DerivationPath::from_str("m/0").unwrap();
+        let desc_key: DescriptorKey<Legacy> = (xprv, path.clone()).to_descriptor_key().unwrap();
+
+        let (desc, _key_map, _valid_networks) = descriptor!(pkh(desc_key)).unwrap();
+        assert_eq!(desc.to_string(), "pkh(tpubD6NzVbkrYhZ4WR7a4vY1VT3khMJMeAxVsfq9TBJyJWrNk247zCJtV7AWf6UJP7rAVsn8NNKdJi3gFyKPTmWZS9iukb91xbn2HbFSMQm2igY/0/*)");
+
+        // as expected this does not compile due to invalid context
+        //let desc_key:DescriptorKey<Segwitv0> = (xprv, path.clone()).to_descriptor_key().unwrap();
+        //let (desc, _key_map, _valid_networks) = descriptor!(pkh(desc_key)).unwrap();
+    }
+
+    #[test]
+    fn test_dsl_modifiers() {
+        let private_key =
+            PrivateKey::from_wif("cSQPHDBwXGjVzWRqAHm6zfvQhaTuj1f2bFH58h55ghbjtFwvmeXR").unwrap();
+        let (descriptor, _, _) =
+            descriptor!(wsh(thresh(2,d:v:older(1),s:pk(private_key),s:pk(private_key)))).unwrap();
+
+        assert_eq!(descriptor.to_string(), "wsh(thresh(2,dv:older(1),s:pk(02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c),s:pk(02e96fe52ef0e22d2f131dd425ce1893073a3c6ad20e8cac36726393dfb4856a4c)))")
+    }
+}

src/descriptor/error.rs

@@ -1,40 +1,82 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Descriptor errors
+
+/// Errors related to the parsing and usage of descriptors
 #[derive(Debug)]
 pub enum Error {
-    InternalError,
-    InvalidPrefix(Vec<u8>),
-    HardenedDerivationOnXpub,
-    MalformedInput,
-    KeyParsingError(String),
-
-    AliasAsPublicKey,
-    KeyHasSecret,
-    Incomplete,
-    MissingAlias(String),
-    InvalidAlias(String),
+    //InternalError,
+    //InvalidPrefix(Vec<u8>),
+    //HardenedDerivationOnXpub,
+    //MalformedInput,
+    /// Invalid HD Key path, such as having a wildcard but a length != 1
+    InvalidHDKeyPath,
 
+    //KeyParsingError(String),
+    /// Error thrown while working with [`keys`](crate::keys)
+    Key(crate::keys::KeyError),
+    /// Error while extracting and manipulating policies
     Policy(crate::descriptor::policy::PolicyError),
 
-    InputIndexDoesntExist,
-    MissingPublicKey,
-    MissingDetails,
-
+    //InputIndexDoesntExist,
+    //MissingPublicKey,
+    //MissingDetails,
+    /// Invalid character found in the descriptor checksum
     InvalidDescriptorCharacter(char),
 
-    CantDeriveWithMiniscript,
-
+    //CantDeriveWithMiniscript,
+    /// BIP32 error
     BIP32(bitcoin::util::bip32::Error),
+    /// Error during base58 decoding
     Base58(bitcoin::util::base58::Error),
+    /// Key-related error
     PK(bitcoin::util::key::Error),
+    /// Miniscript error
     Miniscript(miniscript::Error),
+    /// Hex decoding error
     Hex(bitcoin::hashes::hex::Error),
 }
 
+impl From<crate::keys::KeyError> for Error {
+    fn from(key_error: crate::keys::KeyError) -> Error {
+        match key_error {
+            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
+            crate::keys::KeyError::BIP32(inner) => Error::BIP32(inner),
+            e => Error::Key(e),
+        }
+    }
+}
+
 impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{:?}", self)
     }
 }
 
+impl std::error::Error for Error {}
+
 impl_error!(bitcoin::util::bip32::Error, BIP32);
 impl_error!(bitcoin::util::base58::Error, Base58);
 impl_error!(bitcoin::util::key::Error, PK);

src/descriptor/extended_key.rs

@@ -1,372 +0,0 @@
-use std::fmt::{self, Display};
-use std::str::FromStr;
-
-use bitcoin::hashes::hex::{FromHex, ToHex};
-use bitcoin::secp256k1;
-use bitcoin::util::base58;
-use bitcoin::util::bip32::{
-    ChildNumber, DerivationPath, ExtendedPrivKey, ExtendedPubKey, Fingerprint,
-};
-use bitcoin::PublicKey;
-
-#[allow(unused_imports)]
-use log::{debug, error, info, trace};
-
-#[derive(Copy, Clone, PartialEq, Eq, Debug)]
-pub enum DerivationIndex {
-    Fixed,
-    Normal,
-    Hardened,
-}
-
-impl DerivationIndex {
-    fn as_path(&self, index: u32) -> DerivationPath {
-        match self {
-            DerivationIndex::Fixed => vec![],
-            DerivationIndex::Normal => vec![ChildNumber::Normal { index }],
-            DerivationIndex::Hardened => vec![ChildNumber::Hardened { index }],
-        }
-        .into()
-    }
-}
-
-impl Display for DerivationIndex {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let chars = match *self {
-            Self::Fixed => "",
-            Self::Normal => "/*",
-            Self::Hardened => "/*'",
-        };
-
-        write!(f, "{}", chars)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct DescriptorExtendedKey {
-    pub master_fingerprint: Option<Fingerprint>,
-    pub master_derivation: Option<DerivationPath>,
-    pub pubkey: ExtendedPubKey,
-    pub secret: Option<ExtendedPrivKey>,
-    pub path: DerivationPath,
-    pub final_index: DerivationIndex,
-}
-
-impl DescriptorExtendedKey {
-    pub fn full_path(&self, index: u32) -> DerivationPath {
-        let mut final_path: Vec<ChildNumber> = Vec::new();
-        if let Some(path) = &self.master_derivation {
-            let path_as_vec: Vec<ChildNumber> = path.clone().into();
-            final_path.extend_from_slice(&path_as_vec);
-        }
-        let our_path: Vec<ChildNumber> = self.path_with_index(index).into();
-        final_path.extend_from_slice(&our_path);
-
-        final_path.into()
-    }
-
-    pub fn path_with_index(&self, index: u32) -> DerivationPath {
-        let mut final_path: Vec<ChildNumber> = Vec::new();
-        let our_path: Vec<ChildNumber> = self.path.clone().into();
-        final_path.extend_from_slice(&our_path);
-        let other_path: Vec<ChildNumber> = self.final_index.as_path(index).into();
-        final_path.extend_from_slice(&other_path);
-
-        final_path.into()
-    }
-
-    pub fn derive<C: secp256k1::Verification + secp256k1::Signing>(
-        &self,
-        ctx: &secp256k1::Secp256k1<C>,
-        index: u32,
-    ) -> Result<PublicKey, super::Error> {
-        Ok(self.derive_xpub(ctx, index)?.public_key)
-    }
-
-    pub fn derive_xpub<C: secp256k1::Verification + secp256k1::Signing>(
-        &self,
-        ctx: &secp256k1::Secp256k1<C>,
-        index: u32,
-    ) -> Result<ExtendedPubKey, super::Error> {
-        if let Some(xprv) = self.secret {
-            let derive_priv = xprv.derive_priv(ctx, &self.path_with_index(index))?;
-            Ok(ExtendedPubKey::from_private(ctx, &derive_priv))
-        } else {
-            Ok(self.pubkey.derive_pub(ctx, &self.path_with_index(index))?)
-        }
-    }
-
-    pub fn root_xpub<C: secp256k1::Verification + secp256k1::Signing>(
-        &self,
-        ctx: &secp256k1::Secp256k1<C>,
-    ) -> ExtendedPubKey {
-        if let Some(ref xprv) = self.secret {
-            ExtendedPubKey::from_private(ctx, xprv)
-        } else {
-            self.pubkey
-        }
-    }
-}
-
-impl Display for DescriptorExtendedKey {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(ref fingerprint) = self.master_fingerprint {
-            write!(f, "[{}", fingerprint.to_hex())?;
-            if let Some(ref path) = self.master_derivation {
-                write!(f, "{}", &path.to_string()[1..])?;
-            }
-            write!(f, "]")?;
-        }
-
-        if let Some(xprv) = self.secret {
-            write!(f, "{}", xprv)?
-        } else {
-            write!(f, "{}", self.pubkey)?
-        }
-
-        write!(f, "{}{}", &self.path.to_string()[1..], self.final_index)
-    }
-}
-
-impl FromStr for DescriptorExtendedKey {
-    type Err = super::Error;
-
-    fn from_str(inp: &str) -> Result<DescriptorExtendedKey, Self::Err> {
-        let len = inp.len();
-
-        let (master_fingerprint, master_derivation, offset) = match inp.starts_with("[") {
-            false => (None, None, 0),
-            true => {
-                if inp.len() < 9 {
-                    return Err(super::Error::MalformedInput);
-                }
-
-                let master_fingerprint = &inp[1..9];
-                let close_bracket_index =
-                    &inp[9..].find("]").ok_or(super::Error::MalformedInput)?;
-                let path = if *close_bracket_index > 0 {
-                    Some(DerivationPath::from_str(&format!(
-                        "m{}",
-                        &inp[9..9 + *close_bracket_index]
-                    ))?)
-                } else {
-                    None
-                };
-
-                (
-                    Some(Fingerprint::from_hex(master_fingerprint)?),
-                    path,
-                    9 + *close_bracket_index + 1,
-                )
-            }
-        };
-
-        let (key_range, offset) = match &inp[offset..].find("/") {
-            Some(index) => (offset..offset + *index, offset + *index),
-            None => (offset..len, len),
-        };
-        let data = base58::from_check(&inp[key_range.clone()])?;
-        let secp = secp256k1::Secp256k1::new();
-        let (pubkey, secret) = match &data[0..4] {
-            [0x04u8, 0x88, 0xB2, 0x1E] | [0x04u8, 0x35, 0x87, 0xCF] => {
-                (ExtendedPubKey::from_str(&inp[key_range])?, None)
-            }
-            [0x04u8, 0x88, 0xAD, 0xE4] | [0x04u8, 0x35, 0x83, 0x94] => {
-                let private = ExtendedPrivKey::from_str(&inp[key_range])?;
-                (ExtendedPubKey::from_private(&secp, &private), Some(private))
-            }
-            data => return Err(super::Error::InvalidPrefix(data.into())),
-        };
-
-        let (path, final_index, _) = match &inp[offset..].starts_with("/") {
-            false => (DerivationPath::from(vec![]), DerivationIndex::Fixed, offset),
-            true => {
-                let (all, skip) = match &inp[len - 2..len] {
-                    "/*" => (DerivationIndex::Normal, 2),
-                    "*'" | "*h" => (DerivationIndex::Hardened, 3),
-                    _ => (DerivationIndex::Fixed, 0),
-                };
-
-                if all == DerivationIndex::Hardened && secret.is_none() {
-                    return Err(super::Error::HardenedDerivationOnXpub);
-                }
-
-                (
-                    DerivationPath::from_str(&format!("m{}", &inp[offset..len - skip]))?,
-                    all,
-                    len,
-                )
-            }
-        };
-
-        if secret.is_none()
-            && path.into_iter().any(|child| match child {
-                ChildNumber::Hardened { .. } => true,
-                _ => false,
-            })
-        {
-            return Err(super::Error::HardenedDerivationOnXpub);
-        }
-
-        Ok(DescriptorExtendedKey {
-            master_fingerprint,
-            master_derivation,
-            pubkey,
-            secret,
-            path,
-            final_index,
-        })
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::str::FromStr;
-
-    use bitcoin::hashes::hex::FromHex;
-    use bitcoin::util::bip32::{ChildNumber, DerivationPath};
-
-    use crate::descriptor::*;
-
-    macro_rules! hex_fingerprint {
-        ($hex:expr) => {
-            Fingerprint::from_hex($hex).unwrap()
-        };
-    }
-
-    macro_rules! deriv_path {
-        ($str:expr) => {
-            DerivationPath::from_str($str).unwrap()
-        };
-
-        () => {
-            DerivationPath::from(vec![])
-        };
-    }
-
-    #[test]
-    fn test_derivation_index_fixed() {
-        let index = DerivationIndex::Fixed;
-        assert_eq!(index.as_path(1337), DerivationPath::from(vec![]));
-        assert_eq!(format!("{}", index), "");
-    }
-
-    #[test]
-    fn test_derivation_index_normal() {
-        let index = DerivationIndex::Normal;
-        assert_eq!(
-            index.as_path(1337),
-            DerivationPath::from(vec![ChildNumber::Normal { index: 1337 }])
-        );
-        assert_eq!(format!("{}", index), "/*");
-    }
-
-    #[test]
-    fn test_derivation_index_hardened() {
-        let index = DerivationIndex::Hardened;
-        assert_eq!(
-            index.as_path(1337),
-            DerivationPath::from(vec![ChildNumber::Hardened { index: 1337 }])
-        );
-        assert_eq!(format!("{}", index), "/*'");
-    }
-
-    #[test]
-    fn test_parse_xpub_no_path_fixed() {
-        let key = "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.pubkey.fingerprint(), hex_fingerprint!("31a507b8"));
-        assert_eq!(ek.path, deriv_path!());
-        assert_eq!(ek.final_index, DerivationIndex::Fixed);
-    }
-
-    #[test]
-    fn test_parse_xpub_with_path_fixed() {
-        let key = "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/2/3";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.pubkey.fingerprint(), hex_fingerprint!("31a507b8"));
-        assert_eq!(ek.path, deriv_path!("m/1/2/3"));
-        assert_eq!(ek.final_index, DerivationIndex::Fixed);
-    }
-
-    #[test]
-    fn test_parse_xpub_with_path_normal() {
-        let key = "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/2/3/*";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.pubkey.fingerprint(), hex_fingerprint!("31a507b8"));
-        assert_eq!(ek.path, deriv_path!("m/1/2/3"));
-        assert_eq!(ek.final_index, DerivationIndex::Normal);
-    }
-
-    #[test]
-    #[should_panic(expected = "HardenedDerivationOnXpub")]
-    fn test_parse_xpub_with_path_hardened() {
-        let key = "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*'";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.pubkey.fingerprint(), hex_fingerprint!("31a507b8"));
-        assert_eq!(ek.path, deriv_path!("m/1/2/3"));
-        assert_eq!(ek.final_index, DerivationIndex::Fixed);
-    }
-
-    #[test]
-    fn test_parse_tprv_with_path_hardened() {
-        let key = "tprv8ZgxMBicQKsPduL5QnGihpprdHyypMGi4DhimjtzYemu7se5YQNcZfAPLqXRuGHb5ZX2eTQj62oNqMnyxJ7B7wz54Uzswqw8fFqMVdcmVF7/1/2/3/*'";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert!(ek.secret.is_some());
-        assert_eq!(ek.pubkey.fingerprint(), hex_fingerprint!("5ea4190e"));
-        assert_eq!(ek.path, deriv_path!("m/1/2/3"));
-        assert_eq!(ek.final_index, DerivationIndex::Hardened);
-    }
-
-    #[test]
-    fn test_parse_xpub_master_details() {
-        let key = "[d34db33f/44'/0'/0']xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.master_fingerprint, Some(hex_fingerprint!("d34db33f")));
-        assert_eq!(ek.master_derivation, Some(deriv_path!("m/44'/0'/0'")));
-    }
-
-    #[test]
-    fn test_parse_xpub_master_details_empty_derivation() {
-        let key = "[d34db33f]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.master_fingerprint, Some(hex_fingerprint!("d34db33f")));
-        assert_eq!(ek.master_derivation, None);
-    }
-
-    #[test]
-    #[should_panic(expected = "MalformedInput")]
-    fn test_parse_xpub_short_input() {
-        let key = "[d34d";
-        DescriptorExtendedKey::from_str(key).unwrap();
-    }
-
-    #[test]
-    #[should_panic(expected = "MalformedInput")]
-    fn test_parse_xpub_missing_closing_bracket() {
-        let key = "[d34db33fxpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL";
-        DescriptorExtendedKey::from_str(key).unwrap();
-    }
-
-    #[test]
-    #[should_panic(expected = "InvalidChar")]
-    fn test_parse_xpub_invalid_fingerprint() {
-        let key = "[d34db33z]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL";
-        DescriptorExtendedKey::from_str(key).unwrap();
-    }
-
-    #[test]
-    fn test_xpub_normal_full_path() {
-        let key = "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/2/*";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.full_path(42), deriv_path!("m/1/2/42"));
-    }
-
-    #[test]
-    fn test_xpub_fixed_full_path() {
-        let key = "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/2";
-        let ek = DescriptorExtendedKey::from_str(key).unwrap();
-        assert_eq!(ek.full_path(42), deriv_path!("m/1/2"));
-        assert_eq!(ek.full_path(1337), deriv_path!("m/1/2"));
-    }
-}

src/descriptor/keys.rs

@@ -1,280 +0,0 @@
-use std::fmt;
-use std::str::FromStr;
-
-use bitcoin::secp256k1::{All, Secp256k1};
-use bitcoin::{PrivateKey, PublicKey};
-
-use bitcoin::util::bip32::{
-    ChildNumber, DerivationPath, ExtendedPrivKey, ExtendedPubKey, Fingerprint,
-};
-
-use super::error::Error;
-use super::extended_key::DerivationIndex;
-use super::DescriptorExtendedKey;
-
-#[derive(Debug, Clone)]
-pub struct KeyAlias {
-    alias: String,
-    has_secret: bool,
-}
-
-impl KeyAlias {
-    pub(crate) fn new_boxed(alias: &str, has_secret: bool) -> Box<dyn Key> {
-        Box::new(KeyAlias {
-            alias: alias.into(),
-            has_secret,
-        })
-    }
-}
-
-pub(crate) fn parse_key(string: &str) -> Result<(String, Box<dyn RealKey>), Error> {
-    if let Ok(pk) = PublicKey::from_str(string) {
-        return Ok((string.to_string(), Box::new(pk)));
-    } else if let Ok(sk) = PrivateKey::from_wif(string) {
-        return Ok((string.to_string(), Box::new(sk)));
-    } else if let Ok(ext_key) = DescriptorExtendedKey::from_str(string) {
-        return Ok((string.to_string(), Box::new(ext_key)));
-    }
-
-    return Err(Error::KeyParsingError(string.to_string()));
-}
-
-pub trait Key: std::fmt::Debug + std::fmt::Display {
-    fn as_public_key(&self, secp: &Secp256k1<All>, index: Option<u32>) -> Result<PublicKey, Error>;
-    fn is_fixed(&self) -> bool;
-
-    fn alias(&self) -> Option<&str> {
-        None
-    }
-
-    fn as_secret_key(&self) -> Option<PrivateKey> {
-        None
-    }
-
-    fn xprv(&self) -> Option<ExtendedPrivKey> {
-        None
-    }
-
-    fn full_path(&self, _index: u32) -> Option<DerivationPath> {
-        None
-    }
-
-    fn fingerprint(&self, _secp: &Secp256k1<All>) -> Option<Fingerprint> {
-        None
-    }
-
-    fn has_secret(&self) -> bool {
-        self.xprv().is_some() || self.as_secret_key().is_some()
-    }
-
-    fn public(&self, secp: &Secp256k1<All>) -> Result<Box<dyn RealKey>, Error> {
-        Ok(Box::new(self.as_public_key(secp, None)?))
-    }
-}
-
-pub trait RealKey: Key {
-    fn into_key(&self) -> Box<dyn Key>;
-}
-
-impl<T: RealKey + 'static> From<T> for Box<dyn RealKey> {
-    fn from(key: T) -> Self {
-        Box::new(key)
-    }
-}
-
-impl Key for PublicKey {
-    fn as_public_key(
-        &self,
-        _secp: &Secp256k1<All>,
-        _index: Option<u32>,
-    ) -> Result<PublicKey, Error> {
-        Ok(PublicKey::clone(self))
-    }
-
-    fn is_fixed(&self) -> bool {
-        true
-    }
-}
-
-impl RealKey for PublicKey {
-    fn into_key(&self) -> Box<dyn Key> {
-        Box::new(self.clone())
-    }
-}
-
-impl Key for PrivateKey {
-    fn as_public_key(
-        &self,
-        secp: &Secp256k1<All>,
-        _index: Option<u32>,
-    ) -> Result<PublicKey, Error> {
-        Ok(self.public_key(secp))
-    }
-
-    fn as_secret_key(&self) -> Option<PrivateKey> {
-        Some(PrivateKey::clone(self))
-    }
-
-    fn is_fixed(&self) -> bool {
-        true
-    }
-}
-impl RealKey for PrivateKey {
-    fn into_key(&self) -> Box<dyn Key> {
-        Box::new(self.clone())
-    }
-}
-
-impl Key for DescriptorExtendedKey {
-    fn fingerprint(&self, secp: &Secp256k1<All>) -> Option<Fingerprint> {
-        if let Some(fing) = self.master_fingerprint {
-            Some(fing.clone())
-        } else {
-            Some(self.root_xpub(secp).fingerprint())
-        }
-    }
-
-    fn as_public_key(&self, secp: &Secp256k1<All>, index: Option<u32>) -> Result<PublicKey, Error> {
-        Ok(self.derive_xpub(secp, index.unwrap_or(0))?.public_key)
-    }
-
-    fn public(&self, secp: &Secp256k1<All>) -> Result<Box<dyn RealKey>, Error> {
-        if self.final_index == DerivationIndex::Hardened {
-            return Err(Error::HardenedDerivationOnXpub);
-        }
-
-        if self.xprv().is_none() {
-            return Ok(Box::new(self.clone()));
-        }
-
-        // copy the part of the path that can be derived on the xpub
-        let path = self
-            .path
-            .into_iter()
-            .rev()
-            .take_while(|child| match child {
-                ChildNumber::Normal { .. } => true,
-                _ => false,
-            })
-            .cloned()
-            .collect::<Vec<_>>();
-        // take the prefix that has to be derived on the xprv
-        let master_derivation_add = self
-            .path
-            .into_iter()
-            .take(self.path.as_ref().len() - path.len())
-            .cloned()
-            .collect::<Vec<_>>();
-        let has_derived = !master_derivation_add.is_empty();
-
-        let derived_xprv = self
-            .secret
-            .as_ref()
-            .unwrap()
-            .derive_priv(secp, &master_derivation_add)?;
-        let pubkey = ExtendedPubKey::from_private(secp, &derived_xprv);
-
-        let master_derivation = self
-            .master_derivation
-            .as_ref()
-            .map_or(vec![], |path| path.as_ref().to_vec())
-            .into_iter()
-            .chain(master_derivation_add.into_iter())
-            .collect::<Vec<_>>();
-        let master_derivation = match &master_derivation[..] {
-            &[] => None,
-            child_vec => Some(child_vec.into()),
-        };
-
-        let master_fingerprint = match self.master_fingerprint {
-            Some(desc) => Some(desc.clone()),
-            None if has_derived => Some(self.fingerprint(secp).unwrap()),
-            _ => None,
-        };
-
-        Ok(Box::new(DescriptorExtendedKey {
-            master_fingerprint,
-            master_derivation,
-            pubkey,
-            secret: None,
-            path: path.into(),
-            final_index: self.final_index,
-        }))
-    }
-
-    fn xprv(&self) -> Option<ExtendedPrivKey> {
-        self.secret
-    }
-
-    fn full_path(&self, index: u32) -> Option<DerivationPath> {
-        Some(self.full_path(index))
-    }
-
-    fn is_fixed(&self) -> bool {
-        self.final_index == DerivationIndex::Fixed
-    }
-}
-impl RealKey for DescriptorExtendedKey {
-    fn into_key(&self) -> Box<dyn Key> {
-        Box::new(self.clone())
-    }
-}
-
-impl std::fmt::Display for KeyAlias {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let flag = if self.has_secret { "#" } else { "" };
-
-        write!(f, "{}{}", flag, self.alias)
-    }
-}
-
-impl Key for KeyAlias {
-    fn as_public_key(
-        &self,
-        _secp: &Secp256k1<All>,
-        _index: Option<u32>,
-    ) -> Result<PublicKey, Error> {
-        Err(Error::AliasAsPublicKey)
-    }
-
-    fn is_fixed(&self) -> bool {
-        true
-    }
-
-    fn alias(&self) -> Option<&str> {
-        Some(self.alias.as_str())
-    }
-
-    fn has_secret(&self) -> bool {
-        self.has_secret
-    }
-
-    fn public(&self, _secp: &Secp256k1<All>) -> Result<Box<dyn RealKey>, Error> {
-        Err(Error::AliasAsPublicKey)
-    }
-}
-
-#[derive(Debug, Clone, Hash, PartialEq, PartialOrd, Eq, Ord, Default)]
-pub(crate) struct DummyKey();
-
-impl fmt::Display for DummyKey {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "DummyKey")
-    }
-}
-
-impl std::str::FromStr for DummyKey {
-    type Err = ();
-
-    fn from_str(_: &str) -> Result<Self, Self::Err> {
-        Ok(DummyKey::default())
-    }
-}
-
-impl miniscript::MiniscriptKey for DummyKey {
-    type Hash = DummyKey;
-
-    fn to_pubkeyhash(&self) -> DummyKey {
-        DummyKey::default()
-    }
-}

src/descriptor/template.rs

@@ -0,0 +1,727 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Descriptor templates
+//!
+//! This module contains the definition of various common script templates that are ready to be
+//! used. See the documentation of each template for an example.
+
+use bitcoin::util::bip32;
+use bitcoin::Network;
+
+use miniscript::{Legacy, Segwitv0};
+
+use super::{ExtendedDescriptor, KeyMap, ToWalletDescriptor};
+use crate::keys::{DerivableKey, KeyError, ToDescriptorKey, ValidNetworks};
+use crate::{descriptor, KeychainKind};
+
+/// Type alias for the return type of [`DescriptorTemplate`], [`descriptor!`](crate::descriptor!) and others
+pub type DescriptorTemplateOut = (ExtendedDescriptor, KeyMap, ValidNetworks);
+
+/// Trait for descriptor templates that can be built into a full descriptor
+///
+/// Since [`ToWalletDescriptor`] is implemented for any [`DescriptorTemplate`], they can also be
+/// passed directly to the [`Wallet`](crate::Wallet) constructor.
+///
+/// ## Example
+///
+/// ```
+/// use bdk::keys::{KeyError, ToDescriptorKey};
+/// use bdk::miniscript::Legacy;
+/// use bdk::template::{DescriptorTemplate, DescriptorTemplateOut};
+///
+/// struct MyP2PKH<K: ToDescriptorKey<Legacy>>(K);
+///
+/// impl<K: ToDescriptorKey<Legacy>> DescriptorTemplate for MyP2PKH<K> {
+///     fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+///         Ok(bdk::descriptor!(pkh(self.0))?)
+///     }
+/// }
+/// ```
+pub trait DescriptorTemplate {
+    /// Build the complete descriptor
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError>;
+}
+
+/// Turns a [`DescriptorTemplate`] into a valid wallet descriptor by calling its
+/// [`build`](DescriptorTemplate::build) method
+impl<T: DescriptorTemplate> ToWalletDescriptor for T {
+    fn to_wallet_descriptor(
+        self,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), KeyError> {
+        Ok(self.build()?.to_wallet_descriptor(network)?)
+    }
+}
+
+/// P2PKH template. Expands to a descriptor `pkh(key)`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::P2PKH;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     P2PKH(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_new_address()?.to_string(),
+///     "mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct P2PKH<K: ToDescriptorKey<Legacy>>(pub K);
+
+impl<K: ToDescriptorKey<Legacy>> DescriptorTemplate for P2PKH<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(descriptor!(pkh(self.0))?)
+    }
+}
+
+/// P2WPKH-P2SH template. Expands to a descriptor `sh(wpkh(key))`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::P2WPKH_P2SH;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     P2WPKH_P2SH(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_new_address()?.to_string(),
+///     "2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+#[allow(non_camel_case_types)]
+pub struct P2WPKH_P2SH<K: ToDescriptorKey<Segwitv0>>(pub K);
+
+impl<K: ToDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH_P2SH<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(descriptor!(sh(wpkh(self.0)))?)
+    }
+}
+
+/// P2WPKH template. Expands to a descriptor `wpkh(key)`
+///
+/// ## Example
+///
+/// ```
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::P2WPKH;
+///
+/// let key =
+///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     P2WPKH(key),
+///     None,
+///     Network::Testnet,
+///     MemoryDatabase::default(),
+/// )?;
+///
+/// assert_eq!(
+///     wallet.get_new_address()?.to_string(),
+///     "tb1q4525hmgw265tl3drrl8jjta7ayffu6jf68ltjd"
+/// );
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct P2WPKH<K: ToDescriptorKey<Segwitv0>>(pub K);
+
+impl<K: ToDescriptorKey<Segwitv0>> DescriptorTemplate for P2WPKH<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(descriptor!(wpkh(self.0))?)
+    }
+}
+
+/// BIP44 template. Expands to `pkh(key/44'/0'/0'/{0,1}/*)`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`BIP44Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP44;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP44(key.clone(), KeychainKind::External),
+///     Some(BIP44(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP44<K: DerivableKey<Legacy>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2PKH(legacy::make_bipxx_private(44, self.0, self.1)?).build()?)
+    }
+}
+
+/// BIP44 public template. Expands to `pkh(key/{0,1}/*)`
+///
+/// This assumes that the key used has already been derived with `m/44'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`BIP44`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP44Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP44Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(BIP44Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "pkh([c55b303f/44'/0'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP44Public<K: DerivableKey<Legacy>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Legacy>> DescriptorTemplate for BIP44Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2PKH(legacy::make_bipxx_public(44, self.0, self.1, self.2)?).build()?)
+    }
+}
+
+/// BIP49 template. Expands to `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`BIP49Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP49;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP49(key.clone(), KeychainKind::External),
+///     Some(BIP49(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP49<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH_P2SH(segwit_v0::make_bipxx_private(49, self.0, self.1)?).build()?)
+    }
+}
+
+/// BIP49 public template. Expands to `sh(wpkh(key/{0,1}/*))`
+///
+/// This assumes that the key used has already been derived with `m/49'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`BIP49`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP49Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP49Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(BIP49Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "sh(wpkh([c55b303f/49\'/0\'/0\']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP49Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP49Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH_P2SH(segwit_v0::make_bipxx_public(49, self.0, self.1, self.2)?).build()?)
+    }
+}
+
+/// BIP84 template. Expands to `wpkh(key/84'/0'/0'/{0,1}/*)`
+///
+/// Since there are hardened derivation steps, this template requires a private derivable key (generally a `xprv`/`tprv`).
+///
+/// See [`BIP84Public`] for a template that can work with a `xpub`/`tpub`.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP84;
+///
+/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP84(key.clone(), KeychainKind::External),
+///     Some(BIP84(key, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP84<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH(segwit_v0::make_bipxx_private(84, self.0, self.1)?).build()?)
+    }
+}
+
+/// BIP84 public template. Expands to `wpkh(key/{0,1}/*)`
+///
+/// This assumes that the key used has already been derived with `m/84'/0'/0'`.
+///
+/// This template requires the parent fingerprint to populate correctly the metadata of PSBTs.
+///
+/// See [`BIP84`] for a template that does the full derivation, but requires private data
+/// for the key.
+///
+/// ## Example
+///
+/// ```
+/// # use std::str::FromStr;
+/// # use bdk::bitcoin::{PrivateKey, Network};
+/// # use bdk::{Wallet, OfflineWallet, KeychainKind};
+/// # use bdk::database::MemoryDatabase;
+/// use bdk::template::BIP84Public;
+///
+/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let wallet: OfflineWallet<_> = Wallet::new_offline(
+///     BIP84Public(key.clone(), fingerprint, KeychainKind::External),
+///     Some(BIP84Public(key, fingerprint, KeychainKind::Internal)),
+///     Network::Testnet,
+///     MemoryDatabase::default()
+/// )?;
+///
+/// assert_eq!(wallet.get_new_address()?.to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External)?.unwrap().to_string(), "wpkh([c55b303f/84\'/0\'/0\']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)");
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub struct BIP84Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
+
+impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for BIP84Public<K> {
+    fn build(self) -> Result<DescriptorTemplateOut, KeyError> {
+        Ok(P2WPKH(segwit_v0::make_bipxx_public(84, self.0, self.1, self.2)?).build()?)
+    }
+}
+
+macro_rules! expand_make_bipxx {
+    ( $mod_name:ident, $ctx:ty ) => {
+        mod $mod_name {
+            use super::*;
+
+            pub(super) fn make_bipxx_private<K: DerivableKey<$ctx>>(
+                bip: u32,
+                key: K,
+                keychain: KeychainKind,
+            ) -> Result<impl ToDescriptorKey<$ctx>, KeyError> {
+                let mut derivation_path = Vec::with_capacity(4);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(bip)?);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+                derivation_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+
+                match keychain {
+                    KeychainKind::External => {
+                        derivation_path.push(bip32::ChildNumber::from_normal_idx(0)?)
+                    }
+                    KeychainKind::Internal => {
+                        derivation_path.push(bip32::ChildNumber::from_normal_idx(1)?)
+                    }
+                };
+
+                let derivation_path: bip32::DerivationPath = derivation_path.into();
+
+                Ok((key, derivation_path))
+            }
+            pub(super) fn make_bipxx_public<K: DerivableKey<$ctx>>(
+                bip: u32,
+                key: K,
+                parent_fingerprint: bip32::Fingerprint,
+                keychain: KeychainKind,
+            ) -> Result<impl ToDescriptorKey<$ctx>, KeyError> {
+                let derivation_path: bip32::DerivationPath = match keychain {
+                    KeychainKind::External => vec![bip32::ChildNumber::from_normal_idx(0)?].into(),
+                    KeychainKind::Internal => vec![bip32::ChildNumber::from_normal_idx(1)?].into(),
+                };
+
+                let mut source_path = Vec::with_capacity(3);
+                source_path.push(bip32::ChildNumber::from_hardened_idx(bip)?);
+                source_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+                source_path.push(bip32::ChildNumber::from_hardened_idx(0)?);
+                let source_path: bip32::DerivationPath = source_path.into();
+
+                Ok((key, (parent_fingerprint, source_path), derivation_path))
+            }
+        }
+    };
+}
+
+expand_make_bipxx!(legacy, Legacy);
+expand_make_bipxx!(segwit_v0, Segwitv0);
+
+#[cfg(test)]
+mod test {
+    // test existing descriptor templates, make sure they are expanded to the right descriptors
+
+    use super::*;
+    use crate::descriptor::DescriptorMeta;
+    use crate::keys::{KeyError, ValidNetworks};
+    use bitcoin::hashes::core::str::FromStr;
+    use bitcoin::network::constants::Network::Regtest;
+    use bitcoin::secp256k1::Secp256k1;
+    use bitcoin::util::bip32::ChildNumber;
+    use miniscript::descriptor::{DescriptorPublicKey, DescriptorPublicKeyCtx, KeyMap};
+    use miniscript::Descriptor;
+
+    // verify template descriptor generates expected address(es)
+    fn check(
+        desc: Result<(Descriptor<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError>,
+        is_witness: bool,
+        is_fixed: bool,
+        expected: &[&str],
+    ) {
+        let secp = Secp256k1::new();
+        let deriv_ctx =
+            DescriptorPublicKeyCtx::new(&secp, ChildNumber::from_normal_idx(0).unwrap());
+
+        let (desc, _key_map, _networks) = desc.unwrap();
+        assert_eq!(desc.is_witness(), is_witness);
+        assert_eq!(desc.is_fixed(), is_fixed);
+        for i in 0..expected.len() {
+            let index = i as u32;
+            let child_desc = if desc.is_fixed() {
+                desc.clone()
+            } else {
+                desc.derive(ChildNumber::from_normal_idx(index).unwrap())
+            };
+            let address = child_desc.address(Regtest, deriv_ctx).unwrap();
+            assert_eq!(address.to_string(), *expected.get(i).unwrap());
+        }
+    }
+
+    // P2PKH
+    #[test]
+    fn test_p2ph_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2PKH(prvkey).build(),
+            false,
+            true,
+            &["mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2PKH(pubkey).build(),
+            false,
+            true,
+            &["muZpTpBYhxmRFuCjLc7C6BBDF32C8XVJUi"],
+        );
+    }
+
+    // P2WPKH-P2SH `sh(wpkh(key))`
+    #[test]
+    fn test_p2wphp2sh_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2WPKH_P2SH(prvkey).build(),
+            true,
+            true,
+            &["2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2WPKH_P2SH(pubkey).build(),
+            true,
+            true,
+            &["2N5LiC3CqzxDamRTPG1kiNv1FpNJQ7x28sb"],
+        );
+    }
+
+    // P2WPKH `wpkh(key)`
+    #[test]
+    fn test_p2wph_template() {
+        let prvkey =
+            bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")
+                .unwrap();
+        check(
+            P2WPKH(prvkey).build(),
+            true,
+            true,
+            &["bcrt1q4525hmgw265tl3drrl8jjta7ayffu6jfcwxx9y"],
+        );
+
+        let pubkey = bitcoin::PublicKey::from_str(
+            "03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd",
+        )
+        .unwrap();
+        check(
+            P2WPKH(pubkey).build(),
+            true,
+            true,
+            &["bcrt1qngw83fg8dz0k749cg7k3emc7v98wy0c7azaa6h"],
+        );
+    }
+
+    // BIP44 `pkh(key/44'/0'/0'/{0,1}/*)`
+    #[test]
+    fn test_bip44_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            BIP44(prvkey, KeychainKind::External).build(),
+            false,
+            false,
+            &[
+                "n453VtnjDHPyDt2fDstKSu7A3YCJoHZ5g5",
+                "mvfrrumXgTtwFPWDNUecBBgzuMXhYM7KRP",
+                "mzYvhRAuQqbdSKMVVzXNYyqihgNdRadAUQ",
+            ],
+        );
+        check(
+            BIP44(prvkey, KeychainKind::Internal).build(),
+            false,
+            false,
+            &[
+                "muHF98X9KxEzdKrnFAX85KeHv96eXopaip",
+                "n4hpyLJE5ub6B5Bymv4eqFxS5KjrewSmYR",
+                "mgvkdv1ffmsXd2B1sRKQ5dByK3SzpG42rA",
+            ],
+        );
+    }
+
+    // BIP44 public `pkh(key/{0,1}/*)`
+    #[test]
+    fn test_bip44_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            BIP44Public(pubkey, fingerprint, KeychainKind::External).build(),
+            false,
+            false,
+            &[
+                "miNG7dJTzJqNbFS19svRdTCisC65dsubtR",
+                "n2UqaDbCjWSFJvpC84m3FjUk5UaeibCzYg",
+                "muCPpS6Ue7nkzeJMWDViw7Lkwr92Yc4K8g",
+            ],
+        );
+        check(
+            BIP44Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            false,
+            false,
+            &[
+                "moDr3vJ8wpt5nNxSK55MPq797nXJb2Ru9H",
+                "ms7A1Yt4uTezT2XkefW12AvLoko8WfNJMG",
+                "mhYiyat2rtEnV77cFfQsW32y1m2ceCGHPo",
+            ],
+        );
+    }
+
+    // BIP49 `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
+    #[test]
+    fn test_bip49_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            BIP49(prvkey, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "2N9bCAJXGm168MjVwpkBdNt6ucka3PKVoUV",
+                "2NDckYkqrYyDMtttEav5hB3Bfw9EGAW5HtS",
+                "2NAFTVtksF9T4a97M7nyCjwUBD24QevZ5Z4",
+            ],
+        );
+        check(
+            BIP49(prvkey, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "2NB3pA8PnzJLGV8YEKNDFpbViZv3Bm1K6CG",
+                "2NBiX2Wzxngb5rPiWpUiJQ2uLVB4HBjFD4p",
+                "2NA8ek4CdQ6aMkveYF6AYuEYNrftB47QGTn",
+            ],
+        );
+    }
+
+    // BIP49 public `sh(wpkh(key/{0,1}/*))`
+    #[test]
+    fn test_bip49_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            BIP49Public(pubkey, fingerprint, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt",
+                "2NCTQfJ1sZa3wQ3pPseYRHbaNEpC3AquEfX",
+                "2MveFxAuC8BYPzTybx7FxSzW8HSd8ATT4z7",
+            ],
+        );
+        check(
+            BIP49Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "2NF2vttKibwyxigxtx95Zw8K7JhDbo5zPVJ",
+                "2Mtmyd8taksxNVWCJ4wVvaiss7QPZGcAJuH",
+                "2NBs3CTVYPr1HCzjB4YFsnWCPCtNg8uMEfp",
+            ],
+        );
+    }
+
+    // BIP84 `wpkh(key/84'/0'/0'/{0,1}/*)`
+    #[test]
+    fn test_bip84_template() {
+        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        check(
+            BIP84(prvkey, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "bcrt1qkmvk2nadgplmd57ztld8nf8v2yxkzmdvwtjf8s",
+                "bcrt1qx0v6zgfwe50m4kqc58cqzcyem7ay2sfl3gvqhp",
+                "bcrt1q4h7fq9zhxst6e69p3n882nfj649l7w9g3zccfp",
+            ],
+        );
+        check(
+            BIP84(prvkey, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "bcrt1qtrwtz00wxl69e5xex7amy4xzlxkaefg3gfdkxa",
+                "bcrt1qqqasfhxpkkf7zrxqnkr2sfhn74dgsrc3e3ky45",
+                "bcrt1qpks7n0gq74hsgsz3phn5vuazjjq0f5eqhsgyce",
+            ],
+        );
+    }
+
+    // BIP84 public `wpkh(key/{0,1}/*)`
+    #[test]
+    fn test_bip84_public_template() {
+        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
+        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        check(
+            BIP84Public(pubkey, fingerprint, KeychainKind::External).build(),
+            true,
+            false,
+            &[
+                "bcrt1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2prcdvd0h",
+                "bcrt1q3lncdlwq3lgcaaeyruynjnlccr0ve0kakh6ana",
+                "bcrt1qt9800y6xl3922jy3uyl0z33jh5wfpycyhcylr9",
+            ],
+        );
+        check(
+            BIP84Public(pubkey, fingerprint, KeychainKind::Internal).build(),
+            true,
+            false,
+            &[
+                "bcrt1qm6wqukenh7guu792lj2njgw9n78cmwsy8xy3z2",
+                "bcrt1q694twxtjn4nnrvnyvra769j0a23rllj5c6cgwp",
+                "bcrt1qhlac3c5ranv5w5emlnqs7wxhkxt8maelylcarp",
+            ],
+        );
+    }
+}

src/doctest.rs

@@ -0,0 +1,3 @@
+#[doc(include = "../README.md")]
+#[cfg(doctest)]
+pub struct ReadmeDoctests;

src/error.rs

@@ -1,74 +1,185 @@
-use bitcoin::{OutPoint, Script, Txid};
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
 
+use std::fmt;
+
+use crate::{descriptor, wallet, wallet::address_validator};
+use bitcoin::OutPoint;
+
+/// Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet)
 #[derive(Debug)]
 pub enum Error {
-    KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
-    MissingInputUTXO(usize),
+    /// Wrong number of bytes found when trying to convert to u32
     InvalidU32Bytes(Vec<u8>),
+    /// Generic error
     Generic(String),
+    /// This error is thrown when trying to convert Bare and Public key script to address
     ScriptDoesntHaveAddressForm,
-    SendAllMultipleOutputs,
+    /// Found multiple outputs when `single_recipient` option has been specified
+    SingleRecipientMultipleOutputs,
+    /// `single_recipient` option is selected but neither `drain_wallet` nor `manually_selected_only` are
+    SingleRecipientNoInputs,
+    /// Cannot build a tx without recipients
+    NoRecipients,
+    /// `manually_selected_only` option is selected but no utxo has been passed
+    NoUtxosSelected,
+    /// Output created is under the dust limit, 546 satoshis
     OutputBelowDustLimit(usize),
+    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
     InsufficientFunds,
+    /// Branch and bound coin selection possible attempts with sufficiently big UTXO set could grow
+    /// exponentially, thus a limit is set, and when hit, this error is thrown
+    BnBTotalTriesExceeded,
+    /// Branch and bound coin selection tries to avoid needing a change by finding the right inputs for
+    /// the desired outputs plus fee, if there is not such combination this error is thrown
+    BnBNoExactMatch,
+    /// Happens when trying to spend an UTXO that is not in the internal database
     UnknownUTXO,
-    DifferentTransactions,
-
+    /// Thrown when a tx is not found in the internal database
+    TransactionNotFound,
+    /// Happens when trying to bump a transaction that is already confirmed
+    TransactionConfirmed,
+    /// Trying to replace a tx that has a sequence >= `0xFFFFFFFE`
+    IrreplaceableTransaction,
+    /// When bumping a tx the fee rate requested is lower than required
+    FeeRateTooLow {
+        /// Required fee rate (satoshi/vbyte)
+        required: crate::types::FeeRate,
+    },
+    /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
+    FeeTooLow {
+        /// Required fee absolute value (satoshi)
+        required: u64,
+    },
+    /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
+    /// key in the descriptor must either be a master key itself (having depth = 0) or have an
+    /// explicit origin provided
+    ///
+    /// [`TxBuilder::add_global_xpubs`]: crate::wallet::tx_builder::TxBuilder::add_global_xpubs
+    MissingKeyOrigin(String),
+    /// Error while working with [`keys`](crate::keys)
+    Key(crate::keys::KeyError),
+    /// Descriptor checksum mismatch
     ChecksumMismatch,
-    DifferentDescriptorStructure,
-
-    SpendingPolicyRequired,
+    /// Spending policy is not compatible with this [`KeychainKind`](crate::types::KeychainKind)
+    SpendingPolicyRequired(crate::types::KeychainKind),
+    /// Error while extracting and manipulating policies
     InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
-
-    // Signing errors (expected, received)
-    InputTxidMismatch((Txid, OutPoint)),
-    InputRedeemScriptMismatch((Script, Script)), // scriptPubKey, redeemScript
-    InputWitnessScriptMismatch((Script, Script)), // scriptPubKey, redeemScript
-    InputUnknownSegwitScript(Script),
-    InputMissingWitnessScript(usize),
-    MissingUTXO,
+    /// Signing error
+    Signer(crate::wallet::signer::SignerError),
 
     // Blockchain interface errors
-    Uncapable(crate::blockchain::Capability),
+    /// Thrown when trying to call a method that requires a network connection, [`Wallet::sync`](crate::Wallet::sync) and [`Wallet::broadcast`](crate::Wallet::broadcast)
+    /// This error is thrown when creating the Client for the first time, while recovery attempts are tried
+    /// during the sync
     OfflineClient,
+    /// Progress value must be between `0.0` (included) and `100.0` (included)
     InvalidProgressValue(f32),
+    /// Progress update error (maybe the channel has been closed)
     ProgressUpdateError,
-    MissingCachedAddresses,
+    /// Requested outpoint doesn't exist in the tx (vout greater than available outputs)
     InvalidOutpoint(OutPoint),
 
+    /// Error related to the parsing and usage of descriptors
     Descriptor(crate::descriptor::error::Error),
-
+    /// Error that can be returned to fail the validation of an address
+    AddressValidator(crate::wallet::address_validator::AddressValidatorError),
+    /// Encoding error
     Encode(bitcoin::consensus::encode::Error),
+    /// Miniscript error
+    Miniscript(miniscript::Error),
+    /// BIP32 error
     BIP32(bitcoin::util::bip32::Error),
+    /// An ECDSA error
     Secp256k1(bitcoin::secp256k1::Error),
+    /// Error serializing or deserializing JSON data
     JSON(serde_json::Error),
+    /// Hex decoding error
     Hex(bitcoin::hashes::hex::Error),
+    /// Partially signed bitcoin transaction error
     PSBT(bitcoin::util::psbt::Error),
 
+    //KeyMismatch(bitcoin::secp256k1::PublicKey, bitcoin::secp256k1::PublicKey),
+    //MissingInputUTXO(usize),
+    //InvalidAddressNetwork(Address),
+    //DifferentTransactions,
+    //DifferentDescriptorStructure,
+    //Uncapable(crate::blockchain::Capability),
+    //MissingCachedAddresses,
     #[cfg(feature = "electrum")]
+    /// Electrum client error
     Electrum(electrum_client::Error),
     #[cfg(feature = "esplora")]
+    /// Esplora client error
     Esplora(crate::blockchain::esplora::EsploraError),
+    #[cfg(feature = "compact_filters")]
+    /// Compact filters client error)
+    CompactFilters(crate::blockchain::compact_filters::CompactFiltersError),
     #[cfg(feature = "key-value-db")]
+    /// Sled database error
     Sled(sled::Error),
 }
 
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for Error {}
+
 macro_rules! impl_error {
     ( $from:ty, $to:ident ) => {
-        impl std::convert::From<$from> for Error {
+        impl_error!($from, $to, Error);
+    };
+    ( $from:ty, $to:ident, $impl_for:ty ) => {
+        impl std::convert::From<$from> for $impl_for {
             fn from(err: $from) -> Self {
-                Error::$to(err)
+                <$impl_for>::$to(err)
             }
         }
     };
 }
 
-impl_error!(crate::descriptor::error::Error, Descriptor);
-impl_error!(
-    crate::descriptor::policy::PolicyError,
-    InvalidPolicyPathError
-);
+impl_error!(descriptor::error::Error, Descriptor);
+impl_error!(address_validator::AddressValidatorError, AddressValidator);
+impl_error!(descriptor::policy::PolicyError, InvalidPolicyPathError);
+impl_error!(wallet::signer::SignerError, Signer);
+
+impl From<crate::keys::KeyError> for Error {
+    fn from(key_error: crate::keys::KeyError) -> Error {
+        match key_error {
+            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
+            crate::keys::KeyError::BIP32(inner) => Error::BIP32(inner),
+            crate::keys::KeyError::InvalidChecksum => Error::ChecksumMismatch,
+            e => Error::Key(e),
+        }
+    }
+}
 
 impl_error!(bitcoin::consensus::encode::Error, Encode);
+impl_error!(miniscript::Error, Miniscript);
 impl_error!(bitcoin::util::bip32::Error, BIP32);
 impl_error!(bitcoin::secp256k1::Error, Secp256k1);
 impl_error!(serde_json::Error, JSON);
@@ -81,3 +192,13 @@ impl_error!(electrum_client::Error, Electrum);
 impl_error!(crate::blockchain::esplora::EsploraError, Esplora);
 #[cfg(feature = "key-value-db")]
 impl_error!(sled::Error, Sled);
+
+#[cfg(feature = "compact_filters")]
+impl From<crate::blockchain::compact_filters::CompactFiltersError> for Error {
+    fn from(other: crate::blockchain::compact_filters::CompactFiltersError) -> Self {
+        match other {
+            crate::blockchain::compact_filters::CompactFiltersError::Global(e) => *e,
+            err @ _ => Error::CompactFilters(err),
+        }
+    }
+}

src/keys/bip39.rs

@@ -0,0 +1,173 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! BIP-0039
+
+// TODO: maybe write our own implementation of bip39? Seems stupid to have an extra dependency for
+// something that should be fairly simple to re-implement.
+
+use bitcoin::util::bip32;
+use bitcoin::Network;
+
+use miniscript::ScriptContext;
+
+use bip39::{Language, Mnemonic, MnemonicType, Seed};
+
+use super::{any_network, DerivableKey, DescriptorKey, GeneratableKey, GeneratedKey, KeyError};
+
+/// Type for a BIP39 mnemonic with an optional passphrase
+pub type MnemonicWithPassphrase = (Mnemonic, Option<String>);
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for Seed {
+    fn add_metadata(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let xprv = bip32::ExtendedPrivKey::new_master(Network::Bitcoin, &self.as_bytes())?;
+        let descriptor_key = xprv.add_metadata(source, derivation_path)?;
+
+        // here we must choose one network to build the xpub, but since the bip39 standard doesn't
+        // encode the network, the xpub we create is actually valid everywhere. so we override the
+        // valid networks with `any_network()`.
+        Ok(descriptor_key.override_valid_networks(any_network()))
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for MnemonicWithPassphrase {
+    fn add_metadata(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let (mnemonic, passphrase) = self;
+        let seed = Seed::new(&mnemonic, passphrase.as_deref().unwrap_or(""));
+        seed.add_metadata(source, derivation_path)
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for Mnemonic {
+    fn add_metadata(
+        self,
+        source: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        (self, None).add_metadata(source, derivation_path)
+    }
+}
+
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for Mnemonic {
+    type Entropy = [u8; 32];
+
+    type Options = (MnemonicType, Language);
+    type Error = Option<bip39::ErrorKind>;
+
+    fn generate_with_entropy(
+        (mnemonic_type, language): Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        let entropy = &entropy.as_ref()[..(mnemonic_type.entropy_bits() / 8)];
+        let mnemonic = Mnemonic::from_entropy(entropy, language).map_err(|e| e.downcast().ok())?;
+
+        Ok(GeneratedKey::new(mnemonic, any_network()))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+
+    use bitcoin::util::bip32;
+
+    use bip39::{Language, Mnemonic, MnemonicType};
+
+    use crate::keys::{any_network, GeneratableKey, GeneratedKey};
+
+    #[test]
+    fn test_keys_bip39_mnemonic() {
+        let mnemonic =
+            "aim bunker wash balance finish force paper analyst cabin spoon stable organ";
+        let mnemonic = Mnemonic::from_phrase(mnemonic, Language::English).unwrap();
+        let path = bip32::DerivationPath::from_str("m/44'/0'/0'/0").unwrap();
+
+        let key = (mnemonic, path);
+        let (desc, keys, networks) = crate::descriptor!(wpkh(key)).unwrap();
+        assert_eq!(desc.to_string(), "wpkh([be83839f/44'/0'/0']xpub6DCQ1YcqvZtSwGWMrwHELPehjWV3f2MGZ69yBADTxFEUAoLwb5Mp5GniQK6tTp3AgbngVz9zEFbBJUPVnkG7LFYt8QMTfbrNqs6FNEwAPKA/0/*)");
+        assert_eq!(keys.len(), 1);
+        assert_eq!(networks.len(), 3);
+    }
+
+    #[test]
+    fn test_keys_bip39_mnemonic_passphrase() {
+        let mnemonic =
+            "aim bunker wash balance finish force paper analyst cabin spoon stable organ";
+        let mnemonic = Mnemonic::from_phrase(mnemonic, Language::English).unwrap();
+        let path = bip32::DerivationPath::from_str("m/44'/0'/0'/0").unwrap();
+
+        let key = ((mnemonic, Some("passphrase".into())), path);
+        let (desc, keys, networks) = crate::descriptor!(wpkh(key)).unwrap();
+        assert_eq!(desc.to_string(), "wpkh([8f6cb80c/44'/0'/0']xpub6DWYS8bbihFevy29M4cbw4ZR3P5E12jB8R88gBDWCTCNpYiDHhYWNywrCF9VZQYagzPmsZpxXpytzSoxynyeFr4ZyzheVjnpLKuse4fiwZw/0/*)");
+        assert_eq!(keys.len(), 1);
+        assert_eq!(networks.len(), 3);
+    }
+
+    #[test]
+    fn test_keys_generate_bip39() {
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate_with_entropy(
+                (MnemonicType::Words12, Language::English),
+                crate::keys::test::TEST_ENTROPY,
+            )
+            .unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+        assert_eq!(
+            generated_mnemonic.to_string(),
+            "primary fetch primary fetch primary fetch primary fetch primary fetch primary fever"
+        );
+
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate_with_entropy(
+                (MnemonicType::Words24, Language::English),
+                crate::keys::test::TEST_ENTROPY,
+            )
+            .unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+        assert_eq!(generated_mnemonic.to_string(), "primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary fetch primary foster");
+    }
+
+    #[test]
+    fn test_keys_generate_bip39_random() {
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate((MnemonicType::Words12, Language::English)).unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+
+        let generated_mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+            Mnemonic::generate((MnemonicType::Words24, Language::English)).unwrap();
+        assert_eq!(generated_mnemonic.valid_networks, any_network());
+    }
+}

src/keys/mod.rs

@@ -0,0 +1,747 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Key formats
+
+use std::any::TypeId;
+use std::collections::HashSet;
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::str::FromStr;
+
+use bitcoin::secp256k1;
+
+use bitcoin::util::bip32;
+use bitcoin::{Network, PrivateKey, PublicKey};
+
+pub use miniscript::descriptor::{
+    DescriptorPublicKey, DescriptorSecretKey, DescriptorSinglePriv, DescriptorSinglePub,
+    SortedMultiVec,
+};
+use miniscript::descriptor::{DescriptorXKey, KeyMap};
+pub use miniscript::ScriptContext;
+use miniscript::{Miniscript, Terminal};
+
+use crate::wallet::utils::SecpCtx;
+
+#[cfg(feature = "keys-bip39")]
+#[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
+pub mod bip39;
+
+/// Set of valid networks for a key
+pub type ValidNetworks = HashSet<Network>;
+
+/// Create a set containing mainnet, testnet and regtest
+pub fn any_network() -> ValidNetworks {
+    vec![Network::Bitcoin, Network::Testnet, Network::Regtest]
+        .into_iter()
+        .collect()
+}
+/// Create a set only containing mainnet
+pub fn mainnet_network() -> ValidNetworks {
+    vec![Network::Bitcoin].into_iter().collect()
+}
+/// Create a set containing testnet and regtest
+pub fn test_networks() -> ValidNetworks {
+    vec![Network::Testnet, Network::Regtest]
+        .into_iter()
+        .collect()
+}
+/// Compute the intersection of two sets
+pub fn merge_networks(a: &ValidNetworks, b: &ValidNetworks) -> ValidNetworks {
+    a.intersection(b).cloned().collect()
+}
+
+/// Container for public or secret keys
+#[derive(Debug)]
+pub enum DescriptorKey<Ctx: ScriptContext> {
+    #[doc(hidden)]
+    Public(DescriptorPublicKey, ValidNetworks, PhantomData<Ctx>),
+    #[doc(hidden)]
+    Secret(DescriptorSecretKey, ValidNetworks, PhantomData<Ctx>),
+}
+
+impl<Ctx: ScriptContext> DescriptorKey<Ctx> {
+    /// Create an instance given a public key and a set of valid networks
+    pub fn from_public(public: DescriptorPublicKey, networks: ValidNetworks) -> Self {
+        DescriptorKey::Public(public, networks, PhantomData)
+    }
+
+    /// Create an instance given a secret key and a set of valid networks
+    pub fn from_secret(secret: DescriptorSecretKey, networks: ValidNetworks) -> Self {
+        DescriptorKey::Secret(secret, networks, PhantomData)
+    }
+
+    /// Override the computed set of valid networks
+    pub fn override_valid_networks(self, networks: ValidNetworks) -> Self {
+        match self {
+            DescriptorKey::Public(key, _, _) => DescriptorKey::Public(key, networks, PhantomData),
+            DescriptorKey::Secret(key, _, _) => DescriptorKey::Secret(key, networks, PhantomData),
+        }
+    }
+
+    // This method is used internally by `bdk::fragment!` and `bdk::descriptor!`. It has to be
+    // public because it is effectively called by external crates, once the macros are expanded,
+    // but since it is not meant to be part of the public api we hide it from the docs.
+    #[doc(hidden)]
+    pub fn extract(
+        self,
+        secp: &SecpCtx,
+    ) -> Result<(DescriptorPublicKey, KeyMap, ValidNetworks), KeyError> {
+        match self {
+            DescriptorKey::Public(public, valid_networks, _) => {
+                Ok((public, KeyMap::default(), valid_networks))
+            }
+            DescriptorKey::Secret(secret, valid_networks, _) => {
+                let mut key_map = KeyMap::with_capacity(1);
+
+                let public = secret
+                    .as_public(secp)
+                    .map_err(|e| miniscript::Error::Unexpected(e.to_string()))?;
+                key_map.insert(public.clone(), secret);
+
+                Ok((public, key_map, valid_networks))
+            }
+        }
+    }
+}
+
+/// Enum representation of the known valid [`ScriptContext`]s
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub enum ScriptContextEnum {
+    /// Legacy scripts
+    Legacy,
+    /// Segwitv0 scripts
+    Segwitv0,
+}
+
+impl ScriptContextEnum {
+    /// Returns whether the script context is [`ScriptContextEnum::Legacy`]
+    pub fn is_legacy(&self) -> bool {
+        self == &ScriptContextEnum::Legacy
+    }
+
+    /// Returns whether the script context is [`ScriptContextEnum::Segwitv0`]
+    pub fn is_segwit_v0(&self) -> bool {
+        self == &ScriptContextEnum::Segwitv0
+    }
+}
+
+/// Trait that adds extra useful methods to [`ScriptContext`]s
+pub trait ExtScriptContext: ScriptContext {
+    /// Returns the [`ScriptContext`] as a [`ScriptContextEnum`]
+    fn as_enum() -> ScriptContextEnum;
+
+    /// Returns whether the script context is [`Legacy`](miniscript::Legacy)
+    fn is_legacy() -> bool {
+        Self::as_enum().is_legacy()
+    }
+
+    /// Returns whether the script context is [`Segwitv0`](miniscript::Segwitv0)
+    fn is_segwit_v0() -> bool {
+        Self::as_enum().is_segwit_v0()
+    }
+}
+
+impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
+    fn as_enum() -> ScriptContextEnum {
+        match TypeId::of::<Ctx>() {
+            t if t == TypeId::of::<miniscript::Legacy>() => ScriptContextEnum::Legacy,
+            t if t == TypeId::of::<miniscript::Segwitv0>() => ScriptContextEnum::Segwitv0,
+            _ => unimplemented!("Unknown ScriptContext type"),
+        }
+    }
+}
+
+/// Trait for objects that can be turned into a public or secret [`DescriptorKey`]
+///
+/// The generic type `Ctx` is used to define the context in which the key is valid: some key
+/// formats, like the mnemonics used by Electrum wallets, encode internally whether the wallet is
+/// legacy or segwit. Thus, trying to turn a valid legacy mnemonic into a `DescriptorKey`
+/// that would become part of a segwit descriptor should fail.
+///
+/// For key types that do care about this, the [`ExtScriptContext`] trait provides some useful
+/// methods that can be used to check at runtime which `Ctx` is being used.
+///
+/// For key types that can do this check statically (because they can only work within a
+/// single `Ctx`), the "specialized" trait can be implemented to make the compiler handle the type
+/// checking.
+///
+/// Keys also have control over the networks they support: constructing the return object with
+/// [`DescriptorKey::from_public`] or [`DescriptorKey::from_secret`] allows to specify a set of
+/// [`ValidNetworks`].
+///
+/// ## Examples
+///
+/// Key type valid in any context:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{DescriptorKey, KeyError, ScriptContext, ToDescriptorKey};
+///
+/// pub struct MyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for MyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         self.pubkey.to_descriptor_key()
+///     }
+/// }
+/// ```
+///
+/// Key type that is only valid on mainnet:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{
+///     mainnet_network, DescriptorKey, DescriptorPublicKey, DescriptorSinglePub, KeyError,
+///     ScriptContext, ToDescriptorKey,
+/// };
+///
+/// pub struct MyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for MyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         Ok(DescriptorKey::from_public(
+///             DescriptorPublicKey::SinglePub(DescriptorSinglePub {
+///                 origin: None,
+///                 key: self.pubkey,
+///             }),
+///             mainnet_network(),
+///         ))
+///     }
+/// }
+/// ```
+///
+/// Key type that internally encodes in which context it's valid. The context is checked at runtime:
+///
+/// ```
+/// use bdk::bitcoin::PublicKey;
+///
+/// use bdk::keys::{DescriptorKey, ExtScriptContext, KeyError, ScriptContext, ToDescriptorKey};
+///
+/// pub struct MyKeyType {
+///     is_legacy: bool,
+///     pubkey: PublicKey,
+/// }
+///
+/// impl<Ctx: ScriptContext + 'static> ToDescriptorKey<Ctx> for MyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+///         if Ctx::is_legacy() == self.is_legacy {
+///             self.pubkey.to_descriptor_key()
+///         } else {
+///             Err(KeyError::InvalidScriptContext)
+///         }
+///     }
+/// }
+/// ```
+///
+/// Key type that can only work within [`miniscript::Segwitv0`] context. Only the specialized version
+/// of the trait is implemented.
+///
+/// This example deliberately fails to compile, to demonstrate how the compiler can catch when keys
+/// are misused. In this case, the "segwit-only" key is used to build a `pkh()` descriptor, which
+/// makes the compiler (correctly) fail.
+///
+/// ```compile_fail
+/// use bdk::bitcoin::PublicKey;
+/// use std::str::FromStr;
+///
+/// use bdk::keys::{DescriptorKey, KeyError, ToDescriptorKey};
+///
+/// pub struct MySegwitOnlyKeyType {
+///     pubkey: PublicKey,
+/// }
+///
+/// impl ToDescriptorKey<bdk::miniscript::Segwitv0> for MySegwitOnlyKeyType {
+///     fn to_descriptor_key(self) -> Result<DescriptorKey<bdk::miniscript::Segwitv0>, KeyError> {
+///         self.pubkey.to_descriptor_key()
+///     }
+/// }
+///
+/// let key = MySegwitOnlyKeyType {
+///     pubkey: PublicKey::from_str("...")?,
+/// };
+/// let (descriptor, _, _) = bdk::descriptor!(pkh(key))?;
+/// //                                       ^^^^^ changing this to `wpkh` would make it compile
+///
+/// # Ok::<_, Box<dyn std::error::Error>>(())
+/// ```
+pub trait ToDescriptorKey<Ctx: ScriptContext>: Sized {
+    /// Turn the key into a [`DescriptorKey`] within the requested [`ScriptContext`]
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError>;
+}
+
+/// Trait for keys that can be derived.
+///
+/// When extra metadata are provided, a [`DerivableKey`] can be transofrmed into a
+/// [`DescriptorKey`]: the trait [`ToDescriptorKey`] is automatically implemented
+/// for `(DerivableKey, DerivationPath)` and
+/// `(DerivableKey, KeySource, DerivationPath)` tuples.
+///
+/// For key types that don't encode any indication about the path to use (like bip39), it's
+/// generally recommended to implemented this trait instead of [`ToDescriptorKey`]. The same
+/// rules regarding script context and valid networks apply.
+///
+/// [`DerivationPath`]: (bip32::DerivationPath)
+pub trait DerivableKey<Ctx: ScriptContext> {
+    /// Add a extra metadata, consume `self` and turn it into a [`DescriptorKey`]
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError>;
+}
+
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPubKey {
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorPublicKey::XPub(DescriptorXKey {
+            origin,
+            xkey: self,
+            derivation_path,
+            is_wildcard: true,
+        })
+        .to_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPrivKey {
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::XPrv(DescriptorXKey {
+            origin,
+            xkey: self,
+            derivation_path,
+            is_wildcard: true,
+        })
+        .to_descriptor_key()
+    }
+}
+
+/// Output of a [`GeneratableKey`] key generation
+pub struct GeneratedKey<K, Ctx: ScriptContext> {
+    key: K,
+    valid_networks: ValidNetworks,
+    phantom: PhantomData<Ctx>,
+}
+
+impl<K, Ctx: ScriptContext> GeneratedKey<K, Ctx> {
+    fn new(key: K, valid_networks: ValidNetworks) -> Self {
+        GeneratedKey {
+            key,
+            valid_networks,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Consumes `self` and returns the key
+    pub fn into_key(self) -> K {
+        self.key
+    }
+}
+
+impl<K, Ctx: ScriptContext> Deref for GeneratedKey<K, Ctx> {
+    type Target = K;
+
+    fn deref(&self) -> &Self::Target {
+        &self.key
+    }
+}
+
+// Make generated "derivable" keys themselves "derivable". Also make sure they are assigned the
+// right `valid_networks`.
+impl<Ctx, K> DerivableKey<Ctx> for GeneratedKey<K, Ctx>
+where
+    Ctx: ScriptContext,
+    K: DerivableKey<Ctx>,
+{
+    fn add_metadata(
+        self,
+        origin: Option<bip32::KeySource>,
+        derivation_path: bip32::DerivationPath,
+    ) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let descriptor_key = self.key.add_metadata(origin, derivation_path)?;
+        Ok(descriptor_key.override_valid_networks(self.valid_networks))
+    }
+}
+
+// Make generated keys directly usable in descriptors, and make sure they get assigned the right
+// `valid_networks`.
+impl<Ctx, K> ToDescriptorKey<Ctx> for GeneratedKey<K, Ctx>
+where
+    Ctx: ScriptContext,
+    K: ToDescriptorKey<Ctx>,
+{
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let desc_key = self.key.to_descriptor_key()?;
+        Ok(desc_key.override_valid_networks(self.valid_networks))
+    }
+}
+
+/// Trait for keys that can be generated
+///
+/// The same rules about [`ScriptContext`] and [`ValidNetworks`] from [`ToDescriptorKey`] apply.
+///
+/// This trait is particularly useful when combined with [`DerivableKey`]: if `Self`
+/// implements it, the returned [`GeneratedKey`] will also implement it. The same is true for
+/// [`ToDescriptorKey`]: the generated keys can be directly used in descriptors if `Self` is also
+/// [`ToDescriptorKey`].
+pub trait GeneratableKey<Ctx: ScriptContext>: Sized {
+    /// Type specifying the amount of entropy required e.g. [u8;32]
+    type Entropy: AsMut<[u8]> + Default;
+
+    /// Extra options required by the `generate_with_entropy`
+    type Options;
+    /// Returned error in case of failure
+    type Error: std::fmt::Debug;
+
+    /// Generate a key given the extra options and the entropy
+    fn generate_with_entropy(
+        options: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error>;
+
+    /// Generate a key given the options with a random entropy
+    fn generate(options: Self::Options) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        use rand::{thread_rng, Rng};
+
+        let mut entropy = Self::Entropy::default();
+        thread_rng().fill(entropy.as_mut());
+        Self::generate_with_entropy(options, entropy)
+    }
+}
+
+/// Trait that allows generating a key with the default options
+///
+/// This trait is automatically implemented if the [`GeneratableKey::Options`] implements [`Default`].
+pub trait GeneratableDefaultOptions<Ctx>: GeneratableKey<Ctx>
+where
+    Ctx: ScriptContext,
+    <Self as GeneratableKey<Ctx>>::Options: Default,
+{
+    /// Generate a key with the default options and a given entropy
+    fn generate_with_entropy_default(
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        Self::generate_with_entropy(Default::default(), entropy)
+    }
+
+    /// Generate a key with the default options and a random entropy
+    fn generate_default() -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        Self::generate(Default::default())
+    }
+}
+
+/// Automatic implementation of [`GeneratableDefaultOptions`] for [`GeneratableKey`]s where
+/// `Options` implements `Default`
+impl<Ctx, K> GeneratableDefaultOptions<Ctx> for K
+where
+    Ctx: ScriptContext,
+    K: GeneratableKey<Ctx>,
+    <K as GeneratableKey<Ctx>>::Options: Default,
+{
+}
+
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::ExtendedPrivKey {
+    type Entropy = [u8; 32];
+
+    type Options = ();
+    type Error = bip32::Error;
+
+    fn generate_with_entropy(
+        _: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        // pick a arbitrary network here, but say that we support all of them
+        let xprv = bip32::ExtendedPrivKey::new_master(Network::Bitcoin, entropy.as_ref())?;
+        Ok(GeneratedKey::new(xprv, any_network()))
+    }
+}
+
+/// Options for generating a [`PrivateKey`]
+///
+/// Defaults to creating compressed keys, which save on-chain bytes and fees
+#[derive(Debug, Copy, Clone)]
+pub struct PrivateKeyGenerateOptions {
+    /// Whether the generated key should be "compressed" or not
+    pub compressed: bool,
+}
+
+impl Default for PrivateKeyGenerateOptions {
+    fn default() -> Self {
+        PrivateKeyGenerateOptions { compressed: true }
+    }
+}
+
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for PrivateKey {
+    type Entropy = [u8; secp256k1::constants::SECRET_KEY_SIZE];
+
+    type Options = PrivateKeyGenerateOptions;
+    type Error = bip32::Error;
+
+    fn generate_with_entropy(
+        options: Self::Options,
+        entropy: Self::Entropy,
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        // pick a arbitrary network here, but say that we support all of them
+        let key = secp256k1::SecretKey::from_slice(&entropy)?;
+        let private_key = PrivateKey {
+            compressed: options.compressed,
+            network: Network::Bitcoin,
+            key,
+        };
+
+        Ok(GeneratedKey::new(private_key, any_network()))
+    }
+}
+
+impl<Ctx: ScriptContext, T: DerivableKey<Ctx>> ToDescriptorKey<Ctx> for (T, bip32::DerivationPath) {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        self.0.add_metadata(None, self.1)
+    }
+}
+
+impl<Ctx: ScriptContext, T: DerivableKey<Ctx>> ToDescriptorKey<Ctx>
+    for (T, bip32::KeySource, bip32::DerivationPath)
+{
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        self.0.add_metadata(Some(self.1), self.2)
+    }
+}
+
+fn expand_multi_keys<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<(Vec<DescriptorPublicKey>, KeyMap, ValidNetworks), KeyError> {
+    let (pks, key_maps_networks): (Vec<_>, Vec<_>) = pks
+        .into_iter()
+        .map(|key| Ok::<_, KeyError>(key.to_descriptor_key()?.extract(secp)?))
+        .collect::<Result<Vec<_>, _>>()?
+        .into_iter()
+        .map(|(a, b, c)| (a, (b, c)))
+        .unzip();
+
+    let (key_map, valid_networks) = key_maps_networks.into_iter().fold(
+        (KeyMap::default(), any_network()),
+        |(mut keys_acc, net_acc), (key, net)| {
+            keys_acc.extend(key.into_iter());
+            let net_acc = merge_networks(&net_acc, &net);
+
+            (keys_acc, net_acc)
+        },
+    );
+
+    Ok((pks, key_map, valid_networks))
+}
+
+// Used internally by `bdk::fragment!` to build `pk_k()` fragments
+#[doc(hidden)]
+pub fn make_pk<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    descriptor_key: Pk,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), KeyError> {
+    let (key, key_map, valid_networks) = descriptor_key.to_descriptor_key()?.extract(secp)?;
+
+    Ok((
+        Miniscript::from_ast(Terminal::PkK(key))?,
+        key_map,
+        valid_networks,
+    ))
+}
+
+// Used internally by `bdk::fragment!` to build `multi()` fragments
+#[doc(hidden)]
+pub fn make_multi<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    thresh: usize,
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<(Miniscript<DescriptorPublicKey, Ctx>, KeyMap, ValidNetworks), KeyError> {
+    let (pks, key_map, valid_networks) = expand_multi_keys(pks, secp)?;
+
+    Ok((
+        Miniscript::from_ast(Terminal::Multi(thresh, pks))?,
+        key_map,
+        valid_networks,
+    ))
+}
+
+// Used internally by `bdk::descriptor!` to build `sortedmulti()` fragments
+#[doc(hidden)]
+pub fn make_sortedmulti_inner<Pk: ToDescriptorKey<Ctx>, Ctx: ScriptContext>(
+    thresh: usize,
+    pks: Vec<Pk>,
+    secp: &SecpCtx,
+) -> Result<
+    (
+        SortedMultiVec<DescriptorPublicKey, Ctx>,
+        KeyMap,
+        ValidNetworks,
+    ),
+    KeyError,
+> {
+    let (pks, key_map, valid_networks) = expand_multi_keys(pks, secp)?;
+
+    Ok((SortedMultiVec::new(thresh, pks)?, key_map, valid_networks))
+}
+
+/// The "identity" conversion is used internally by some `bdk::fragment`s
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for DescriptorKey<Ctx> {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        Ok(self)
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for DescriptorPublicKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let networks = match self {
+            DescriptorPublicKey::SinglePub(_) => any_network(),
+            DescriptorPublicKey::XPub(DescriptorXKey { xkey, .. })
+                if xkey.network == Network::Bitcoin =>
+            {
+                mainnet_network()
+            }
+            _ => test_networks(),
+        };
+
+        Ok(DescriptorKey::from_public(self, networks))
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for PublicKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorPublicKey::SinglePub(DescriptorSinglePub {
+            key: self,
+            origin: None,
+        })
+        .to_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for DescriptorSecretKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        let networks = match &self {
+            DescriptorSecretKey::SinglePriv(sk) if sk.key.network == Network::Bitcoin => {
+                mainnet_network()
+            }
+            DescriptorSecretKey::XPrv(DescriptorXKey { xkey, .. })
+                if xkey.network == Network::Bitcoin =>
+            {
+                mainnet_network()
+            }
+            _ => test_networks(),
+        };
+
+        Ok(DescriptorKey::from_secret(self, networks))
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for &'_ str {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::from_str(self)
+            .map_err(|e| KeyError::Message(e.to_string()))?
+            .to_descriptor_key()
+    }
+}
+
+impl<Ctx: ScriptContext> ToDescriptorKey<Ctx> for PrivateKey {
+    fn to_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
+        DescriptorSecretKey::SinglePriv(DescriptorSinglePriv {
+            key: self,
+            origin: None,
+        })
+        .to_descriptor_key()
+    }
+}
+
+/// Errors thrown while working with [`keys`](crate::keys)
+#[derive(Debug)]
+pub enum KeyError {
+    /// The key cannot exist in the given script context
+    InvalidScriptContext,
+    /// The key is not valid for the given network
+    InvalidNetwork,
+    /// The key has an invalid checksum
+    InvalidChecksum,
+
+    /// Custom error message
+    Message(String),
+
+    /// BIP32 error
+    BIP32(bitcoin::util::bip32::Error),
+    /// Miniscript error
+    Miniscript(miniscript::Error),
+}
+
+impl_error!(miniscript::Error, Miniscript, KeyError);
+impl_error!(bitcoin::util::bip32::Error, BIP32, KeyError);
+
+impl std::fmt::Display for KeyError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for KeyError {}
+
+#[cfg(test)]
+pub mod test {
+    use bitcoin::util::bip32;
+
+    use super::*;
+
+    pub const TEST_ENTROPY: [u8; 32] = [0xAA; 32];
+
+    #[test]
+    fn test_keys_generate_xprv() {
+        let generated_xprv: GeneratedKey<_, miniscript::Segwitv0> =
+            bip32::ExtendedPrivKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+
+        assert_eq!(generated_xprv.valid_networks, any_network());
+        assert_eq!(generated_xprv.to_string(), "xprv9s21ZrQH143K4Xr1cJyqTvuL2FWR8eicgY9boWqMBv8MDVUZ65AXHnzBrK1nyomu6wdcabRgmGTaAKawvhAno1V5FowGpTLVx3jxzE5uk3Q");
+    }
+
+    #[test]
+    fn test_keys_generate_wif() {
+        let generated_wif: GeneratedKey<_, miniscript::Segwitv0> =
+            bitcoin::PrivateKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+
+        assert_eq!(generated_wif.valid_networks, any_network());
+        assert_eq!(
+            generated_wif.to_string(),
+            "L2wTu6hQrnDMiFNWA5na6jB12ErGQqtXwqpSL7aWquJaZG8Ai3ch"
+        );
+    }
+}

src/lib.rs

@@ -1,3 +1,209 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// rustdoc will warn if there are missing docs
+#![warn(missing_docs)]
+// only enables the `doc_cfg` feature when
+// the `docsrs` configuration attribute is defined
+#![cfg_attr(docsrs, feature(doc_cfg))]
+// only enables the nightly `external_doc` feature when
+// `test-md-docs` is enabled
+#![cfg_attr(feature = "test-md-docs", feature(external_doc))]
+
+//! A modern, lightweight, descriptor-based wallet library written in Rust.
+//!
+//! # About
+//!
+//! The BDK library aims to be the core building block for Bitcoin wallets of any kind.
+//!
+//! * It uses [Miniscript](https://github.com/rust-bitcoin/rust-miniscript) to support descriptors with generalized conditions. This exact same library can be used to build
+//!   single-sig wallets, multisigs, timelocked contracts and more.
+//! * It supports multiple blockchain backends and databases, allowing developers to choose exactly what's right for their projects.
+//! * It is built to be cross-platform: the core logic works on desktop, mobile, and even WebAssembly.
+//! * It is very easy to extend: developers can implement customized logic for blockchain backends, databases, signers, coin selection, and more, without having to fork and modify this library.
+//!
+//! # A Tour of BDK
+//!
+//! BDK consists of a number of modules that provide a range of functionality
+//! essential for implementing descriptor based Bitcoin wallet applications in Rust. In this
+//! section, we will take a brief tour of BDK, summarizing the major APIs and
+//! their uses.
+//!
+//! The easiest way to get started is to add bdk to your dependencies with the default features.
+//! The default features include a simple key-value database ([`sled`](sled)) to cache
+//! blockchain data and an [electrum](https://docs.rs/electrum-client/) blockchain client to
+//! interact with the bitcoin P2P network.
+//!
+//! ```toml
+//! bdk = "0.2.0"
+//! ```
+//!
+//! ## Sync the balance of a descriptor
+//!
+//! ### Example
+//! ```ignore
+//! use bdk::Wallet;
+//! use bdk::database::MemoryDatabase;
+//! use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+//!
+//! use bdk::electrum_client::Client;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+//!     let wallet = Wallet::new(
+//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!         ElectrumBlockchain::from(client)
+//!     )?;
+//!
+//!     wallet.sync(noop_progress(), None)?;
+//!
+//!     println!("Descriptor balance: {} SAT", wallet.get_balance()?);
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Generate a few addresses
+//!
+//! ### Example
+//! ```
+//! use bdk::{Wallet, OfflineWallet};
+//! use bdk::database::MemoryDatabase;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let wallet: OfflineWallet<_> = Wallet::new_offline(
+//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!     )?;
+//!
+//!     println!("Address #0: {}", wallet.get_new_address()?);
+//!     println!("Address #1: {}", wallet.get_new_address()?);
+//!     println!("Address #2: {}", wallet.get_new_address()?);
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Create a transaction
+//!
+//! ### Example
+//! ```ignore
+//! use base64::decode;
+//! use bdk::{FeeRate, TxBuilder, Wallet};
+//! use bdk::database::MemoryDatabase;
+//! use bdk::blockchain::{noop_progress, ElectrumBlockchain};
+//!
+//! use bdk::electrum_client::Client;
+//!
+//! use bitcoin::consensus::serialize;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let client = Client::new("ssl://electrum.blockstream.info:60002")?;
+//!     let wallet = Wallet::new(
+//!         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!         ElectrumBlockchain::from(client)
+//!     )?;
+//!
+//!     wallet.sync(noop_progress(), None)?;
+//!
+//!     let send_to = wallet.get_new_address()?;
+//!     let (psbt, details) = wallet.create_tx(
+//!         TxBuilder::with_recipients(vec![(send_to.script_pubkey(), 50_000)])
+//!             .enable_rbf()
+//!             .do_not_spend_change()
+//!             .fee_rate(FeeRate::from_sat_per_vb(5.0))
+//!     )?;
+//!
+//!     println!("Transaction details: {:#?}", details);
+//!     println!("Unsigned PSBT: {}", base64::encode(&serialize(&psbt)));
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Sign a transaction
+//!
+//! ### Example
+//! ```ignore
+//! use base64::decode;
+//! use bdk::{Wallet, OfflineWallet};
+//! use bdk::database::MemoryDatabase;
+//!
+//! use bitcoin::consensus::deserialize;
+//!
+//! fn main() -> Result<(), bdk::Error> {
+//!     let wallet: OfflineWallet<_> = Wallet::new_offline(
+//!         "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)",
+//!         Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"),
+//!         bitcoin::Network::Testnet,
+//!         MemoryDatabase::default(),
+//!     )?;
+//!
+//!     let psbt = "...";
+//!     let psbt = deserialize(&base64::decode(psbt).unwrap())?;
+//!
+//!     let (signed_psbt, finalized) = wallet.sign(psbt, None)?;
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # Feature flags
+//!
+//! BDK uses a set of [feature flags](https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section)
+//! to reduce the amount of compiled code by allowing projects to only enable the features they need.
+//! By default, BDK enables two internal features, `key-value-db` and `electrum`.
+//!
+//! If you are new to BDK we recommended that you use the default features which will enable
+//! basic descriptor wallet functionality. More advanced users can disable the `default` features
+//! (`--no-default-features`) and build the BDK library with only the features you need.
+
+//! Below is a list of the available feature flags and the additional functionality they provide.
+//!
+//! * `all-keys`: all features for working with bitcoin keys
+//! * `async-interface`: async functions in bdk traits
+//! * `cli-utils`: utilities for creating a command line interface wallet
+//! * `keys-bip39`: [BIP-39](https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki) mnemonic codes for generating deterministic keys
+//!
+//! ## Internal features
+//!
+//! These features do not expose any new API, but influence internal implementation aspects of
+//! BDK.
+//!
+//! * `compact_filters`: [`compact_filters`](crate::blockchain::compact_filters) client protocol for interacting with the bitcoin P2P network
+//! * `electrum`: [`electrum`](crate::blockchain::electrum) client protocol for interacting with electrum servers
+//! * `esplora`: [`esplora`](crate::blockchain::esplora) client protocol for interacting with blockstream [electrs](https://github.com/Blockstream/electrs) servers
+//! * `key-value-db`: key value [`database`](crate::database) based on [`sled`](crate::sled) for caching blockchain data
+
 pub extern crate bitcoin;
 extern crate log;
 pub extern crate miniscript;
@@ -5,22 +211,24 @@ extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
-#[cfg(test)]
+#[cfg(feature = "keys-bip39")]
+extern crate bip39;
+
+#[cfg(any(target_arch = "wasm32", feature = "async-interface"))]
+#[macro_use]
+extern crate async_trait;
+#[macro_use]
+extern crate bdk_macros;
+
+#[cfg(feature = "compact_filters")]
 #[macro_use]
 extern crate lazy_static;
 
-#[macro_use]
-extern crate async_trait;
-
 #[cfg(feature = "electrum")]
 pub extern crate electrum_client;
-#[cfg(feature = "electrum")]
-pub use electrum_client::client::Client;
 
 #[cfg(feature = "esplora")]
 pub extern crate reqwest;
-#[cfg(feature = "esplora")]
-pub use blockchain::esplora::EsploraBlockchain;
 
 #[cfg(feature = "key-value-db")]
 pub extern crate sled;
@@ -28,17 +236,36 @@ pub extern crate sled;
 #[cfg(feature = "cli-utils")]
 pub mod cli;
 
+#[allow(unused_imports)]
+#[cfg(test)]
 #[macro_use]
-pub mod error;
+extern crate testutils;
+#[allow(unused_imports)]
+#[cfg(test)]
+#[macro_use]
+extern crate testutils_macros;
+#[allow(unused_imports)]
+#[cfg(test)]
+#[macro_use]
+extern crate serial_test;
+
+#[macro_use]
+pub(crate) mod error;
 pub mod blockchain;
 pub mod database;
 pub mod descriptor;
-#[cfg(feature = "multiparty")]
-pub mod multiparty;
-pub mod psbt;
-pub mod signer;
-pub mod types;
+#[cfg(feature = "test-md-docs")]
+mod doctest;
+pub mod keys;
+pub(crate) mod psbt;
+pub(crate) mod types;
 pub mod wallet;
 
-pub use descriptor::ExtendedDescriptor;
+pub use descriptor::template;
+pub use descriptor::HDKeyPaths;
+pub use error::Error;
+pub use types::*;
+pub use wallet::address_validator;
+pub use wallet::signer;
+pub use wallet::tx_builder::TxBuilder;
 pub use wallet::{OfflineWallet, Wallet};

src/multiparty/mod.rs

@@ -1,231 +0,0 @@
-use std::cell::RefCell;
-use std::collections::BTreeMap;
-
-use bitcoin::secp256k1::Secp256k1;
-
-use crate::descriptor::error::Error;
-use crate::descriptor::keys::{parse_key, DummyKey, Key, KeyAlias, RealKey};
-use crate::descriptor::{ExtendedDescriptor, MiniscriptExtractPolicy, Policy, StringDescriptor};
-
-pub trait ParticipantType: Default {
-    fn validate_aliases(aliases: Vec<&String>) -> Result<(), Error>;
-}
-
-#[derive(Default)]
-pub struct Coordinator {}
-impl ParticipantType for Coordinator {
-    fn validate_aliases(aliases: Vec<&String>) -> Result<(), Error> {
-        if aliases.into_iter().any(|a| a == "[PEER]") {
-            Err(Error::InvalidAlias("[PEER]".into()))
-        } else {
-            Ok(())
-        }
-    }
-}
-
-#[derive(Default)]
-pub struct Peer;
-impl ParticipantType for Peer {
-    fn validate_aliases(aliases: Vec<&String>) -> Result<(), Error> {
-        if !aliases.into_iter().any(|a| a == "[PEER]") {
-            Err(Error::MissingAlias("[PEER]".into()))
-        } else {
-            Ok(())
-        }
-    }
-}
-
-pub struct Participant<T: ParticipantType> {
-    descriptor: StringDescriptor,
-    parsed_keys: BTreeMap<String, Box<dyn Key>>,
-    received_keys: BTreeMap<String, Box<dyn RealKey>>,
-
-    _data: T,
-}
-
-impl<T: ParticipantType> Participant<T> {
-    pub fn new(sd: StringDescriptor) -> Result<Self, Error> {
-        let parsed_keys = Self::parse_keys(&sd, vec![]);
-
-        T::validate_aliases(parsed_keys.keys().collect())?;
-
-        Ok(Participant {
-            descriptor: sd,
-            parsed_keys,
-            received_keys: Default::default(),
-            _data: Default::default(),
-        })
-    }
-
-    fn parse_keys(
-        sd: &StringDescriptor,
-        with_secrets: Vec<&str>,
-    ) -> BTreeMap<String, Box<dyn Key>> {
-        let keys: RefCell<BTreeMap<String, Box<dyn Key>>> = RefCell::new(BTreeMap::new());
-
-        let translatefpk = |string: &String| -> Result<_, Error> {
-            let (key, parsed) = match parse_key(string) {
-                Ok((key, parsed)) => (key, parsed.into_key()),
-                Err(_) => (
-                    string.clone(),
-                    KeyAlias::new_boxed(string.as_str(), with_secrets.contains(&string.as_str())),
-                ),
-            };
-            keys.borrow_mut().insert(key, parsed);
-
-            Ok(DummyKey::default())
-        };
-        let translatefpkh = |string: &String| -> Result<_, Error> {
-            let (key, parsed) = match parse_key(string) {
-                Ok((key, parsed)) => (key, parsed.into_key()),
-                Err(_) => (
-                    string.clone(),
-                    KeyAlias::new_boxed(string.as_str(), with_secrets.contains(&string.as_str())),
-                ),
-            };
-            keys.borrow_mut().insert(key, parsed);
-
-            Ok(DummyKey::default())
-        };
-
-        sd.translate_pk(translatefpk, translatefpkh).unwrap();
-
-        keys.into_inner()
-    }
-
-    pub fn policy_for(&self, with_secrets: Vec<&str>) -> Result<Option<Policy>, Error> {
-        let keys = Self::parse_keys(&self.descriptor, with_secrets);
-        self.descriptor.extract_policy(&keys)
-    }
-
-    fn _missing_keys(&self) -> Vec<&String> {
-        self.parsed_keys
-            .keys()
-            .filter(|k| !self.received_keys.contains_key(*k))
-            .collect()
-    }
-
-    pub fn completed(&self) -> bool {
-        self._missing_keys().is_empty()
-    }
-
-    pub fn finalize(self) -> Result<ExtendedDescriptor, Error> {
-        if !self.completed() {
-            return Err(Error::Incomplete);
-        }
-
-        let translatefpk = |string: &String| -> Result<_, Error> {
-            Ok(format!(
-                "{}",
-                self.received_keys
-                    .get(string)
-                    .expect(&format!("Missing key: `{}`", string))
-            ))
-        };
-        let translatefpkh = |string: &String| -> Result<_, Error> {
-            Ok(format!(
-                "{}",
-                self.received_keys
-                    .get(string)
-                    .expect(&format!("Missing key: `{}`", string))
-            ))
-        };
-
-        let internal = self.descriptor.translate_pk(translatefpk, translatefpkh)?;
-
-        Ok(ExtendedDescriptor {
-            internal,
-            keys: self.received_keys,
-            ctx: Secp256k1::gen_new(),
-        })
-    }
-}
-
-impl Participant<Coordinator> {
-    pub fn descriptor(&self) -> &StringDescriptor {
-        &self.descriptor
-    }
-
-    pub fn add_key(&mut self, alias: &str, key: Box<dyn RealKey>) -> Result<(), Error> {
-        // TODO: check network
-
-        if key.has_secret() {
-            return Err(Error::KeyHasSecret);
-        }
-
-        self.received_keys.insert(alias.into(), key);
-
-        Ok(())
-    }
-
-    pub fn received_keys(&self) -> Vec<&String> {
-        self.received_keys.keys().collect()
-    }
-
-    pub fn missing_keys(&self) -> Vec<&String> {
-        self._missing_keys()
-    }
-
-    pub fn descriptor_for(&self, alias: &str) -> Result<StringDescriptor, Error> {
-        if !self.parsed_keys.contains_key(alias) {
-            return Err(Error::MissingAlias(alias.into()));
-        }
-
-        let map_name = |s: &String| {
-            if s == alias {
-                "[PEER]".into()
-            } else {
-                s.into()
-            }
-        };
-
-        let translatefpk = |string: &String| -> Result<_, Error> { Ok(map_name(string)) };
-        let translatefpkh = |string: &String| -> Result<_, Error> { Ok(map_name(string)) };
-
-        Ok(self.descriptor.translate_pk(translatefpk, translatefpkh)?)
-    }
-
-    pub fn get_map(&self) -> Result<BTreeMap<String, String>, Error> {
-        if !self.completed() {
-            return Err(Error::Incomplete);
-        }
-
-        Ok(self
-            .received_keys
-            .iter()
-            .map(|(k, v)| (k.into(), format!("{}", v)))
-            .collect())
-    }
-}
-
-impl Participant<Peer> {
-    pub fn policy(&self) -> Result<Option<Policy>, Error> {
-        self.policy_for(vec!["[PEER]"])
-    }
-
-    pub fn use_key(&mut self, key: Box<dyn RealKey>) -> Result<(), Error> {
-        let secp = Secp256k1::gen_new();
-        self.received_keys
-            .insert("[PEER]".into(), key.public(&secp)?);
-
-        Ok(())
-    }
-
-    pub fn my_key(&mut self) -> Option<&Box<dyn RealKey>> {
-        self.received_keys.get("[PEER]".into())
-    }
-
-    pub fn apply_map(mut self, map: BTreeMap<String, String>) -> Result<ExtendedDescriptor, Error> {
-        let mut parsed_map: BTreeMap<_, _> = map
-            .into_iter()
-            .map(|(k, v)| -> Result<_, Error> {
-                let (_, parsed) = parse_key(&v)?;
-                Ok((k, parsed))
-            })
-            .collect::<Result<_, _>>()?;
-
-        self.received_keys.append(&mut parsed_map);
-
-        self.finalize()
-    }
-}

src/psbt/mod.rs

@@ -1,271 +1,52 @@
-use std::collections::BTreeMap;
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
 
-use bitcoin::hashes::{hash160, Hash};
-use bitcoin::util::bip143::SighashComponents;
-use bitcoin::util::bip32::{DerivationPath, ExtendedPrivKey, Fingerprint};
-use bitcoin::util::psbt;
-use bitcoin::{PrivateKey, PublicKey, Script, SigHashType, Transaction};
+use bitcoin::util::psbt::PartiallySignedTransaction as PSBT;
+use bitcoin::TxOut;
 
-use bitcoin::secp256k1::{self, All, Message, Secp256k1};
-
-#[allow(unused_imports)]
-use log::{debug, error, info, trace};
-
-use miniscript::{BitcoinSig, MiniscriptKey, Satisfier};
-
-use crate::descriptor::ExtendedDescriptor;
-use crate::error::Error;
-use crate::signer::Signer;
-
-pub mod utils;
-
-pub struct PSBTSatisfier<'a> {
-    input: &'a psbt::Input,
-    assume_height_reached: bool,
-    create_height: Option<u32>,
-    current_height: Option<u32>,
+pub trait PSBTUtils {
+    fn get_utxo_for(&self, input_index: usize) -> Option<TxOut>;
 }
 
-impl<'a> PSBTSatisfier<'a> {
-    pub fn new(
-        input: &'a psbt::Input,
-        assume_height_reached: bool,
-        create_height: Option<u32>,
-        current_height: Option<u32>,
-    ) -> Self {
-        PSBTSatisfier {
-            input,
-            assume_height_reached,
-            create_height,
-            current_height,
+impl PSBTUtils for PSBT {
+    fn get_utxo_for(&self, input_index: usize) -> Option<TxOut> {
+        let tx = &self.global.unsigned_tx;
+
+        if input_index >= tx.input.len() {
+            return None;
         }
-    }
-}
 
-impl<'a> PSBTSatisfier<'a> {
-    fn parse_sig(rawsig: &Vec<u8>) -> Option<BitcoinSig> {
-        let (flag, sig) = rawsig.split_last().unwrap();
-        let flag = bitcoin::SigHashType::from_u32(*flag as u32);
-        let sig = match secp256k1::Signature::from_der(sig) {
-            Ok(sig) => sig,
-            Err(..) => return None,
-        };
-        Some((sig, flag))
-    }
-}
-
-// TODO: also support hash preimages through the "unknown" section of PSBT
-impl<'a> Satisfier<bitcoin::PublicKey> for PSBTSatisfier<'a> {
-    // from https://docs.rs/miniscript/0.12.0/src/miniscript/psbt/mod.rs.html#96
-    fn lookup_sig(&self, pk: &bitcoin::PublicKey) -> Option<BitcoinSig> {
-        debug!("lookup_sig: {}", pk);
-
-        if let Some(rawsig) = self.input.partial_sigs.get(pk) {
-            Self::parse_sig(&rawsig)
+        if let Some(input) = self.inputs.get(input_index) {
+            if let Some(wit_utxo) = &input.witness_utxo {
+                Some(wit_utxo.clone())
+            } else if let Some(in_tx) = &input.non_witness_utxo {
+                Some(in_tx.output[tx.input[input_index].previous_output.vout as usize].clone())
+            } else {
+                None
+            }
         } else {
             None
         }
     }
-
-    fn lookup_pkh_pk(&self, hash: &hash160::Hash) -> Option<bitcoin::PublicKey> {
-        debug!("lookup_pkh_pk: {}", hash);
-
-        for (pk, _) in &self.input.partial_sigs {
-            if &pk.to_pubkeyhash() == hash {
-                return Some(*pk);
-            }
-        }
-
-        None
-    }
-
-    fn lookup_pkh_sig(&self, hash: &hash160::Hash) -> Option<(bitcoin::PublicKey, BitcoinSig)> {
-        debug!("lookup_pkh_sig: {}", hash);
-
-        for (pk, sig) in &self.input.partial_sigs {
-            if &pk.to_pubkeyhash() == hash {
-                return match Self::parse_sig(&sig) {
-                    Some(bitcoinsig) => Some((*pk, bitcoinsig)),
-                    None => None,
-                };
-            }
-        }
-
-        None
-    }
-
-    fn check_older(&self, height: u32) -> bool {
-        // TODO: also check if `nSequence` right
-        debug!("check_older: {}", height);
-
-        if let Some(current_height) = self.current_height {
-            // TODO: test >= / >
-            current_height as u64 >= self.create_height.unwrap_or(0) as u64 + height as u64
-        } else {
-            self.assume_height_reached
-        }
-    }
-
-    fn check_after(&self, height: u32) -> bool {
-        // TODO: also check if `nLockTime` is right
-        debug!("check_after: {}", height);
-
-        if let Some(current_height) = self.current_height {
-            current_height > height
-        } else {
-            self.assume_height_reached
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct PSBTSigner<'a> {
-    tx: &'a Transaction,
-    secp: Secp256k1<All>,
-
-    // psbt: &'b psbt::PartiallySignedTransaction,
-    extended_keys: BTreeMap<Fingerprint, ExtendedPrivKey>,
-    private_keys: BTreeMap<PublicKey, PrivateKey>,
-}
-
-impl<'a> PSBTSigner<'a> {
-    pub fn from_descriptor(tx: &'a Transaction, desc: &ExtendedDescriptor) -> Result<Self, Error> {
-        let secp = Secp256k1::gen_new();
-
-        let mut extended_keys = BTreeMap::new();
-        for xprv in desc.get_xprv() {
-            let fing = xprv.fingerprint(&secp);
-            extended_keys.insert(fing, xprv);
-        }
-
-        let mut private_keys = BTreeMap::new();
-        for privkey in desc.get_secret_keys() {
-            let pubkey = privkey.public_key(&secp);
-            private_keys.insert(pubkey, privkey);
-        }
-
-        Ok(PSBTSigner {
-            tx,
-            secp,
-            extended_keys,
-            private_keys,
-        })
-    }
-
-    pub fn extend(&mut self, mut other: PSBTSigner) -> Result<(), Error> {
-        if self.tx.txid() != other.tx.txid() {
-            return Err(Error::DifferentTransactions);
-        }
-
-        self.extended_keys.append(&mut other.extended_keys);
-        self.private_keys.append(&mut other.private_keys);
-
-        Ok(())
-    }
-
-    // TODO: temporary
-    pub fn all_public_keys(&self) -> impl IntoIterator<Item = &PublicKey> {
-        self.private_keys.keys()
-    }
-}
-
-impl<'a> Signer for PSBTSigner<'a> {
-    fn sig_legacy_from_fingerprint(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        fingerprint: &Fingerprint,
-        path: &DerivationPath,
-        script: &Script,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        self.extended_keys
-            .get(fingerprint)
-            .map_or(Ok(None), |xprv| {
-                let privkey = xprv.derive_priv(&self.secp, path)?;
-                // let derived_pubkey = secp256k1::PublicKey::from_secret_key(&self.secp, &privkey.private_key.key);
-
-                let hash = self.tx.signature_hash(index, script, sighash.as_u32());
-
-                let signature = self.secp.sign(
-                    &Message::from_slice(&hash.into_inner()[..])?,
-                    &privkey.private_key.key,
-                );
-
-                Ok(Some((signature, sighash)))
-            })
-    }
-
-    fn sig_legacy_from_pubkey(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        public_key: &PublicKey,
-        script: &Script,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        self.private_keys
-            .get(public_key)
-            .map_or(Ok(None), |privkey| {
-                let hash = self.tx.signature_hash(index, script, sighash.as_u32());
-
-                let signature = self
-                    .secp
-                    .sign(&Message::from_slice(&hash.into_inner()[..])?, &privkey.key);
-
-                Ok(Some((signature, sighash)))
-            })
-    }
-
-    fn sig_segwit_from_fingerprint(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        fingerprint: &Fingerprint,
-        path: &DerivationPath,
-        script: &Script,
-        value: u64,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        self.extended_keys
-            .get(fingerprint)
-            .map_or(Ok(None), |xprv| {
-                let privkey = xprv.derive_priv(&self.secp, path)?;
-
-                let hash = SighashComponents::new(self.tx).sighash_all(
-                    &self.tx.input[index],
-                    script,
-                    value,
-                );
-
-                let signature = self.secp.sign(
-                    &Message::from_slice(&hash.into_inner()[..])?,
-                    &privkey.private_key.key,
-                );
-
-                Ok(Some((signature, sighash)))
-            })
-    }
-
-    fn sig_segwit_from_pubkey(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        public_key: &PublicKey,
-        script: &Script,
-        value: u64,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        self.private_keys
-            .get(public_key)
-            .map_or(Ok(None), |privkey| {
-                let hash = SighashComponents::new(self.tx).sighash_all(
-                    &self.tx.input[index],
-                    script,
-                    value,
-                );
-
-                let signature = self
-                    .secp
-                    .sign(&Message::from_slice(&hash.into_inner()[..])?, &privkey.key);
-
-                Ok(Some((signature, sighash)))
-            })
-    }
 }

src/psbt/utils.rs

@@ -1,28 +0,0 @@
-use bitcoin::util::psbt::PartiallySignedTransaction as PSBT;
-use bitcoin::TxOut;
-
-pub trait PSBTUtils {
-    fn get_utxo_for(&self, input_index: usize) -> Option<TxOut>;
-}
-
-impl PSBTUtils for PSBT {
-    fn get_utxo_for(&self, input_index: usize) -> Option<TxOut> {
-        let tx = &self.global.unsigned_tx;
-
-        if input_index >= tx.input.len() {
-            return None;
-        }
-
-        if let Some(input) = self.inputs.get(input_index) {
-            if let Some(wit_utxo) = &input.witness_utxo {
-                Some(wit_utxo.clone())
-            } else if let Some(in_tx) = &input.non_witness_utxo {
-                Some(in_tx.output[tx.input[input_index].previous_output.vout as usize].clone())
-            } else {
-                None
-            }
-        } else {
-            None
-        }
-    }
-}

src/signer.rs

@@ -1,87 +0,0 @@
-use bitcoin::util::bip32::{DerivationPath, Fingerprint};
-use bitcoin::{PublicKey, Script, SigHashType};
-
-use miniscript::miniscript::satisfy::BitcoinSig;
-
-use crate::error::Error;
-
-pub trait Signer {
-    fn sig_legacy_from_fingerprint(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        fingerprint: &Fingerprint,
-        path: &DerivationPath,
-        script: &Script,
-    ) -> Result<Option<BitcoinSig>, Error>;
-    fn sig_legacy_from_pubkey(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        public_key: &PublicKey,
-        script: &Script,
-    ) -> Result<Option<BitcoinSig>, Error>;
-
-    fn sig_segwit_from_fingerprint(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        fingerprint: &Fingerprint,
-        path: &DerivationPath,
-        script: &Script,
-        value: u64,
-    ) -> Result<Option<BitcoinSig>, Error>;
-    fn sig_segwit_from_pubkey(
-        &self,
-        index: usize,
-        sighash: SigHashType,
-        public_key: &PublicKey,
-        script: &Script,
-        value: u64,
-    ) -> Result<Option<BitcoinSig>, Error>;
-}
-
-#[allow(dead_code)]
-impl dyn Signer {
-    fn sig_legacy_from_fingerprint(
-        &self,
-        _index: usize,
-        _sighash: SigHashType,
-        _fingerprint: &Fingerprint,
-        _path: &DerivationPath,
-        _script: &Script,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        Ok(None)
-    }
-    fn sig_legacy_from_pubkey(
-        &self,
-        _index: usize,
-        _sighash: SigHashType,
-        _public_key: &PublicKey,
-        _script: &Script,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        Ok(None)
-    }
-
-    fn sig_segwit_from_fingerprint(
-        &self,
-        _index: usize,
-        _sighash: SigHashType,
-        _fingerprint: &Fingerprint,
-        _path: &DerivationPath,
-        _script: &Script,
-        _value: u64,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        Ok(None)
-    }
-    fn sig_segwit_from_pubkey(
-        &self,
-        _index: usize,
-        _sighash: SigHashType,
-        _public_key: &PublicKey,
-        _script: &Script,
-        _value: u64,
-    ) -> Result<Option<BitcoinSig>, Error> {
-        Ok(None)
-    }
-}

src/types.rs

@@ -1,3 +1,27 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 use std::convert::AsRef;
 
 use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
@@ -5,43 +29,93 @@ use bitcoin::hash_types::Txid;
 
 use serde::{Deserialize, Serialize};
 
-// TODO serde flatten?
-#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ScriptType {
+/// Types of keychains
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum KeychainKind {
+    /// External
     External = 0,
+    /// Internal, usually used for change outputs
     Internal = 1,
 }
 
-impl ScriptType {
+impl KeychainKind {
+    /// Return [`KeychainKind`] as a byte
     pub fn as_byte(&self) -> u8 {
         match self {
-            ScriptType::External => 'e' as u8,
-            ScriptType::Internal => 'i' as u8,
+            KeychainKind::External => b'e',
+            KeychainKind::Internal => b'i',
         }
     }
 }
 
-impl AsRef<[u8]> for ScriptType {
+impl AsRef<[u8]> for KeychainKind {
     fn as_ref(&self) -> &[u8] {
         match self {
-            ScriptType::External => b"e",
-            ScriptType::Internal => b"i",
+            KeychainKind::External => b"e",
+            KeychainKind::Internal => b"i",
         }
     }
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub struct UTXO {
-    pub outpoint: OutPoint,
-    pub txout: TxOut,
+/// Fee rate
+#[derive(Debug, Copy, Clone, PartialEq, PartialOrd)]
+// Internally stored as satoshi/vbyte
+pub struct FeeRate(f32);
+
+impl FeeRate {
+    /// Create a new instance of [`FeeRate`] given a float fee rate in btc/kvbytes
+    pub fn from_btc_per_kvb(btc_per_kvb: f32) -> Self {
+        FeeRate(btc_per_kvb * 1e5)
+    }
+
+    /// Create a new instance of [`FeeRate`] given a float fee rate in satoshi/vbyte
+    pub fn from_sat_per_vb(sat_per_vb: f32) -> Self {
+        FeeRate(sat_per_vb)
+    }
+
+    /// Create a new [`FeeRate`] with the default min relay fee value
+    pub fn default_min_relay_fee() -> Self {
+        FeeRate(1.0)
+    }
+
+    /// Return the value as satoshi/vbyte
+    pub fn as_sat_vb(&self) -> f32 {
+        self.0
+    }
 }
 
+impl std::default::Default for FeeRate {
+    fn default() -> Self {
+        FeeRate::default_min_relay_fee()
+    }
+}
+
+/// A wallet unspent output
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct UTXO {
+    /// Reference to a transaction output
+    pub outpoint: OutPoint,
+    /// Transaction output
+    pub txout: TxOut,
+    /// Type of keychain
+    pub keychain: KeychainKind,
+}
+
+/// A wallet transaction
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
 pub struct TransactionDetails {
+    /// Optional transaction
     pub transaction: Option<Transaction>,
+    /// Transaction id
     pub txid: Txid,
+    /// Timestamp
     pub timestamp: u64,
+    /// Received value (sats)
     pub received: u64,
+    /// Sent value (sats)
     pub sent: u64,
+    /// Fee value (sats)
+    pub fees: u64,
+    /// Confirmed in block height, `None` means unconfirmed
     pub height: Option<u32>,
 }

src/wallet/address_validator.rs

@@ -0,0 +1,167 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Address validation callbacks
+//!
+//! The typical usage of those callbacks is for displaying the newly-generated address on a
+//! hardware wallet, so that the user can cross-check its correctness.
+//!
+//! More generally speaking though, these callbacks can also be used to "do something" every time
+//! an address is generated, without necessarily checking or validating it.
+//!
+//! An address validator can be attached to a [`Wallet`](super::Wallet) by using the
+//! [`Wallet::add_address_validator`](super::Wallet::add_address_validator) method, and
+//! whenever a new address is generated (either explicitly by the user with
+//! [`Wallet::get_new_address`](super::Wallet::get_new_address) or internally to create a change
+//! address) all the attached validators will be polled, in sequence. All of them must complete
+//! successfully to continue.
+//!
+//! ## Example
+//!
+//! ```
+//! # use std::sync::Arc;
+//! # use bitcoin::*;
+//! # use bdk::address_validator::*;
+//! # use bdk::database::*;
+//! # use bdk::*;
+//! struct PrintAddressAndContinue;
+//!
+//! impl AddressValidator for PrintAddressAndContinue {
+//!     fn validate(
+//!         &self,
+//!         keychain: KeychainKind,
+//!         hd_keypaths: &HDKeyPaths,
+//!         script: &Script
+//!     ) -> Result<(), AddressValidatorError> {
+//!         let address = Address::from_script(script, Network::Testnet)
+//!             .as_ref()
+//!             .map(Address::to_string)
+//!             .unwrap_or(script.to_string());
+//!         println!("New address of type {:?}: {}", keychain, address);
+//!         println!("HD keypaths: {:#?}", hd_keypaths);
+//!
+//!         Ok(())
+//!     }
+//! }
+//!
+//! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)";
+//! let mut wallet: OfflineWallet<_> = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
+//! wallet.add_address_validator(Arc::new(PrintAddressAndContinue));
+//!
+//! let address = wallet.get_new_address()?;
+//! println!("Address: {}", address);
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use std::fmt;
+
+use bitcoin::Script;
+
+use crate::descriptor::HDKeyPaths;
+use crate::types::KeychainKind;
+
+/// Errors that can be returned to fail the validation of an address
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum AddressValidatorError {
+    /// User rejected the address
+    UserRejected,
+    /// Network connection error
+    ConnectionError,
+    /// Network request timeout error
+    TimeoutError,
+    /// Invalid script
+    InvalidScript,
+    /// A custom error message
+    Message(String),
+}
+
+impl fmt::Display for AddressValidatorError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for AddressValidatorError {}
+
+/// Trait to build address validators
+///
+/// All the address validators attached to a wallet with [`Wallet::add_address_validator`](super::Wallet::add_address_validator) will be polled
+/// every time an address (external or internal) is generated by the wallet. Errors returned in the
+/// validator will be propagated up to the original caller that triggered the address generation.
+///
+/// For a usage example see [this module](crate::address_validator)'s documentation.
+pub trait AddressValidator: Send + Sync {
+    /// Validate or inspect an address
+    fn validate(
+        &self,
+        keychain: KeychainKind,
+        hd_keypaths: &HDKeyPaths,
+        script: &Script,
+    ) -> Result<(), AddressValidatorError>;
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::Arc;
+
+    use super::*;
+    use crate::wallet::test::{get_funded_wallet, get_test_wpkh};
+    use crate::wallet::TxBuilder;
+
+    struct TestValidator;
+    impl AddressValidator for TestValidator {
+        fn validate(
+            &self,
+            _keychain: KeychainKind,
+            _hd_keypaths: &HDKeyPaths,
+            _script: &bitcoin::Script,
+        ) -> Result<(), AddressValidatorError> {
+            Err(AddressValidatorError::InvalidScript)
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "InvalidScript")]
+    fn test_address_validator_external() {
+        let (mut wallet, _, _) = get_funded_wallet(get_test_wpkh());
+        wallet.add_address_validator(Arc::new(TestValidator));
+
+        wallet.get_new_address().unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "InvalidScript")]
+    fn test_address_validator_internal() {
+        let (mut wallet, descriptors, _) = get_funded_wallet(get_test_wpkh());
+        wallet.add_address_validator(Arc::new(TestValidator));
+
+        let addr = testutils!(@external descriptors, 10);
+        wallet
+            .create_tx(TxBuilder::with_recipients(vec![(
+                addr.script_pubkey(),
+                25_000,
+            )]))
+            .unwrap();
+    }
+}

src/wallet/coin_selection.rs

@@ -0,0 +1,997 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Coin selection
+//!
+//! This module provides the trait [`CoinSelectionAlgorithm`] that can be implemented to
+//! define custom coin selection algorithms.
+//!
+//! The coin selection algorithm is not globally part of a [`Wallet`](super::Wallet), instead it
+//! is selected whenever a [`Wallet::create_tx`](super::Wallet::create_tx) call is made, through
+//! the use of the [`TxBuilder`] structure, specifically with
+//! [`TxBuilder::coin_selection`](super::tx_builder::TxBuilder::coin_selection) method.
+//!
+//! The [`DefaultCoinSelectionAlgorithm`] selects the default coin selection algorithm that
+//! [`TxBuilder`] uses, if it's not explicitly overridden.
+//!
+//! [`TxBuilder`]: super::tx_builder::TxBuilder
+//!
+//! ## Example
+//!
+//! ```no_run
+//! # use std::str::FromStr;
+//! # use bitcoin::*;
+//! # use bdk::wallet::coin_selection::*;
+//! # use bdk::database::Database;
+//! # use bdk::*;
+//! # const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4 + 1) * 4;
+//! #[derive(Debug)]
+//! struct AlwaysSpendEverything;
+//!
+//! impl<D: Database> CoinSelectionAlgorithm<D> for AlwaysSpendEverything {
+//!     fn coin_select(
+//!         &self,
+//!         database: &D,
+//!         required_utxos: Vec<(UTXO, usize)>,
+//!         optional_utxos: Vec<(UTXO, usize)>,
+//!         fee_rate: FeeRate,
+//!         amount_needed: u64,
+//!         fee_amount: f32,
+//!     ) -> Result<CoinSelectionResult, bdk::Error> {
+//!         let mut selected_amount = 0;
+//!         let mut additional_weight = 0;
+//!         let all_utxos_selected = required_utxos
+//!             .into_iter().chain(optional_utxos)
+//!             .scan((&mut selected_amount, &mut additional_weight), |(selected_amount, additional_weight), (utxo, weight)| {
+//!                 **selected_amount += utxo.txout.value;
+//!                 **additional_weight += TXIN_BASE_WEIGHT + weight;
+//!
+//!                 Some(utxo)
+//!             })
+//!             .collect::<Vec<_>>();
+//!         let additional_fees = additional_weight as f32 * fee_rate.as_sat_vb() / 4.0;
+//!
+//!         if (fee_amount + additional_fees).ceil() as u64 + amount_needed > selected_amount {
+//!             return Err(bdk::Error::InsufficientFunds);
+//!         }
+//!
+//!         Ok(CoinSelectionResult {
+//!             selected: all_utxos_selected,
+//!             selected_amount,
+//!             fee_amount: fee_amount + additional_fees,
+//!         })
+//!     }
+//! }
+//!
+//! # let wallet: OfflineWallet<_> = Wallet::new_offline("", None, Network::Testnet, bdk::database::MemoryDatabase::default())?;
+//! // create wallet, sync, ...
+//!
+//! let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+//! let (psbt, details) = wallet.create_tx(
+//!     TxBuilder::with_recipients(vec![(to_address.script_pubkey(), 50_000)])
+//!         .coin_selection(AlwaysSpendEverything),
+//! )?;
+//!
+//! // inspect, sign, broadcast, ...
+//!
+//! # Ok::<(), bdk::Error>(())
+//! ```
+
+use crate::database::Database;
+use crate::error::Error;
+use crate::types::{FeeRate, UTXO};
+
+use rand::seq::SliceRandom;
+#[cfg(not(test))]
+use rand::thread_rng;
+#[cfg(test)]
+use rand::{rngs::StdRng, SeedableRng};
+
+/// Default coin selection algorithm used by [`TxBuilder`](super::tx_builder::TxBuilder) if not
+/// overridden
+#[cfg(not(test))]
+pub type DefaultCoinSelectionAlgorithm = BranchAndBoundCoinSelection;
+#[cfg(test)]
+pub type DefaultCoinSelectionAlgorithm = LargestFirstCoinSelection; // make the tests more predictable
+
+// Base weight of a Txin, not counting the weight needed for satisfying it.
+// prev_txid (32 bytes) + prev_vout (4 bytes) + sequence (4 bytes) + script_len (1 bytes)
+pub(crate) const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4 + 1) * 4;
+
+/// Result of a successful coin selection
+#[derive(Debug)]
+pub struct CoinSelectionResult {
+    /// List of outputs selected for use as inputs
+    pub selected: Vec<UTXO>,
+    /// Sum of the selected inputs' value
+    pub selected_amount: u64,
+    /// Total fee amount in satoshi
+    pub fee_amount: f32,
+}
+
+/// Trait for generalized coin selection algorithms
+///
+/// This trait can be implemented to make the [`Wallet`](super::Wallet) use a customized coin
+/// selection algorithm when it creates transactions.
+///
+/// For an example see [this module](crate::wallet::coin_selection)'s documentation.
+pub trait CoinSelectionAlgorithm<D: Database>: std::fmt::Debug {
+    /// Perform the coin selection
+    ///
+    /// - `database`: a reference to the wallet's database that can be used to lookup additional
+    ///               details for a specific UTXO
+    /// - `required_utxos`: the utxos that must be spent regardless of `amount_needed` with their
+    ///                     weight cost
+    /// - `optional_utxos`: the remaining available utxos to satisfy `amount_needed` with their
+    ///                     weight cost
+    /// - `fee_rate`: fee rate to use
+    /// - `amount_needed`: the amount in satoshi to select
+    /// - `fee_amount`: the amount of fees in satoshi already accumulated from adding outputs and
+    ///                 the transaction's header
+    fn coin_select(
+        &self,
+        database: &D,
+        required_utxos: Vec<(UTXO, usize)>,
+        optional_utxos: Vec<(UTXO, usize)>,
+        fee_rate: FeeRate,
+        amount_needed: u64,
+        fee_amount: f32,
+    ) -> Result<CoinSelectionResult, Error>;
+}
+
+/// Simple and dumb coin selection
+///
+/// This coin selection algorithm sorts the available UTXOs by value and then picks them starting
+/// from the largest ones until the required amount is reached.
+#[derive(Debug, Default)]
+pub struct LargestFirstCoinSelection;
+
+impl<D: Database> CoinSelectionAlgorithm<D> for LargestFirstCoinSelection {
+    fn coin_select(
+        &self,
+        _database: &D,
+        required_utxos: Vec<(UTXO, usize)>,
+        mut optional_utxos: Vec<(UTXO, usize)>,
+        fee_rate: FeeRate,
+        amount_needed: u64,
+        mut fee_amount: f32,
+    ) -> Result<CoinSelectionResult, Error> {
+        let calc_fee_bytes = |wu| (wu as f32) * fee_rate.as_sat_vb() / 4.0;
+
+        log::debug!(
+            "amount_needed = `{}`, fee_amount = `{}`, fee_rate = `{:?}`",
+            amount_needed,
+            fee_amount,
+            fee_rate
+        );
+
+        // We put the "required UTXOs" first and make sure the optional UTXOs are sorted,
+        // initially smallest to largest, before being reversed with `.rev()`.
+        let utxos = {
+            optional_utxos.sort_unstable_by_key(|(utxo, _)| utxo.txout.value);
+            required_utxos
+                .into_iter()
+                .map(|utxo| (true, utxo))
+                .chain(optional_utxos.into_iter().rev().map(|utxo| (false, utxo)))
+        };
+
+        // Keep including inputs until we've got enough.
+        // Store the total input value in selected_amount and the total fee being paid in fee_amount
+        let mut selected_amount = 0;
+        let selected = utxos
+            .scan(
+                (&mut selected_amount, &mut fee_amount),
+                |(selected_amount, fee_amount), (must_use, (utxo, weight))| {
+                    if must_use || **selected_amount < amount_needed + (fee_amount.ceil() as u64) {
+                        **fee_amount += calc_fee_bytes(TXIN_BASE_WEIGHT + weight);
+                        **selected_amount += utxo.txout.value;
+
+                        log::debug!(
+                            "Selected {}, updated fee_amount = `{}`",
+                            utxo.outpoint,
+                            fee_amount
+                        );
+
+                        Some(utxo)
+                    } else {
+                        None
+                    }
+                },
+            )
+            .collect::<Vec<_>>();
+
+        if selected_amount < amount_needed + (fee_amount.ceil() as u64) {
+            return Err(Error::InsufficientFunds);
+        }
+
+        Ok(CoinSelectionResult {
+            selected,
+            fee_amount,
+            selected_amount,
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+// Adds fee information to an UTXO.
+struct OutputGroup {
+    utxo: UTXO,
+    // weight needed to satisfy the UTXO, as described in `Descriptor::max_satisfaction_weight`
+    satisfaction_weight: usize,
+    // Amount of fees for spending a certain utxo, calculated using a certain FeeRate
+    fee: f32,
+    // The effective value of the UTXO, i.e., the utxo value minus the fee for spending it
+    effective_value: i64,
+}
+
+impl OutputGroup {
+    fn new(utxo: UTXO, satisfaction_weight: usize, fee_rate: FeeRate) -> Self {
+        let fee = (TXIN_BASE_WEIGHT + satisfaction_weight) as f32 / 4.0 * fee_rate.as_sat_vb();
+        let effective_value = utxo.txout.value as i64 - fee.ceil() as i64;
+        OutputGroup {
+            utxo,
+            satisfaction_weight,
+            effective_value,
+            fee,
+        }
+    }
+}
+
+/// Branch and bound coin selection
+///
+/// Code adapted from Bitcoin Core's implementation and from Mark Erhardt Master's Thesis: <http://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf>
+#[derive(Debug)]
+pub struct BranchAndBoundCoinSelection {
+    size_of_change: u64,
+}
+
+impl Default for BranchAndBoundCoinSelection {
+    fn default() -> Self {
+        Self {
+            // P2WPKH cost of change -> value (8 bytes) + script len (1 bytes) + script (22 bytes)
+            size_of_change: 8 + 1 + 22,
+        }
+    }
+}
+
+impl BranchAndBoundCoinSelection {
+    /// Create new instance with target size for change output
+    pub fn new(size_of_change: u64) -> Self {
+        Self { size_of_change }
+    }
+}
+
+const BNB_TOTAL_TRIES: usize = 100_000;
+
+impl<D: Database> CoinSelectionAlgorithm<D> for BranchAndBoundCoinSelection {
+    fn coin_select(
+        &self,
+        _database: &D,
+        required_utxos: Vec<(UTXO, usize)>,
+        optional_utxos: Vec<(UTXO, usize)>,
+        fee_rate: FeeRate,
+        amount_needed: u64,
+        fee_amount: f32,
+    ) -> Result<CoinSelectionResult, Error> {
+        // Mapping every (UTXO, usize) to an output group
+        let required_utxos: Vec<OutputGroup> = required_utxos
+            .into_iter()
+            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .collect();
+
+        // Mapping every (UTXO, usize) to an output group.
+        // Filtering UTXOs with an effective_value < 0, as the fee paid for
+        // adding them is more than their value
+        let optional_utxos: Vec<OutputGroup> = optional_utxos
+            .into_iter()
+            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .filter(|u| u.effective_value > 0)
+            .collect();
+
+        let curr_value = required_utxos
+            .iter()
+            .fold(0, |acc, x| acc + x.effective_value as u64);
+
+        let curr_available_value = optional_utxos
+            .iter()
+            .fold(0, |acc, x| acc + x.effective_value as u64);
+
+        let actual_target = fee_amount.ceil() as u64 + amount_needed;
+        let cost_of_change = self.size_of_change as f32 * fee_rate.as_sat_vb();
+
+        if curr_available_value + curr_value < actual_target {
+            return Err(Error::InsufficientFunds);
+        }
+
+        Ok(self
+            .bnb(
+                required_utxos.clone(),
+                optional_utxos.clone(),
+                curr_value,
+                curr_available_value,
+                actual_target,
+                fee_amount,
+                cost_of_change,
+            )
+            .unwrap_or_else(|_| {
+                self.single_random_draw(
+                    required_utxos,
+                    optional_utxos,
+                    curr_value,
+                    actual_target,
+                    fee_amount,
+                )
+            }))
+    }
+}
+
+impl BranchAndBoundCoinSelection {
+    // TODO: make this more Rust-onic :)
+    // (And perhpaps refactor with less arguments?)
+    #[allow(clippy::too_many_arguments)]
+    fn bnb(
+        &self,
+        required_utxos: Vec<OutputGroup>,
+        mut optional_utxos: Vec<OutputGroup>,
+        mut curr_value: u64,
+        mut curr_available_value: u64,
+        actual_target: u64,
+        fee_amount: f32,
+        cost_of_change: f32,
+    ) -> Result<CoinSelectionResult, Error> {
+        // current_selection[i] will contain true if we are using optional_utxos[i],
+        // false otherwise. Note that current_selection.len() could be less than
+        // optional_utxos.len(), it just means that we still haven't decided if we should keep
+        // certain optional_utxos or not.
+        let mut current_selection: Vec<bool> = Vec::with_capacity(optional_utxos.len());
+
+        // Sort the utxo_pool
+        optional_utxos.sort_unstable_by_key(|a| a.effective_value);
+        optional_utxos.reverse();
+
+        // Contains the best selection we found
+        let mut best_selection = Vec::new();
+        let mut best_selection_value = None;
+
+        // Depth First search loop for choosing the UTXOs
+        for _ in 0..BNB_TOTAL_TRIES {
+            // Conditions for starting a backtrack
+            let mut backtrack = false;
+            // Cannot possibly reach target with the amount remaining in the curr_available_value,
+            // or the selected value is out of range.
+            // Go back and try other branch
+            if curr_value + curr_available_value < actual_target
+                || curr_value > actual_target + cost_of_change as u64
+            {
+                backtrack = true;
+            } else if curr_value >= actual_target {
+                // Selected value is within range, there's no point in going forward. Start
+                // backtracking
+                backtrack = true;
+
+                // If we found a solution better than the previous one, or if there wasn't previous
+                // solution, update the best solution
+                if best_selection_value.is_none() || curr_value < best_selection_value.unwrap() {
+                    best_selection = current_selection.clone();
+                    best_selection_value = Some(curr_value);
+                }
+
+                // If we found a perfect match, break here
+                if curr_value == actual_target {
+                    break;
+                }
+            }
+
+            // Backtracking, moving backwards
+            if backtrack {
+                // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed.
+                while let Some(false) = current_selection.last() {
+                    current_selection.pop();
+                    curr_available_value +=
+                        optional_utxos[current_selection.len()].effective_value as u64;
+                }
+
+                if current_selection.last_mut().is_none() {
+                    // We have walked back to the first utxo and no branch is untraversed. All solutions searched
+                    // If best selection is empty, then there's no exact match
+                    if best_selection.is_empty() {
+                        return Err(Error::BnBNoExactMatch);
+                    }
+                    break;
+                }
+
+                if let Some(c) = current_selection.last_mut() {
+                    // Output was included on previous iterations, try excluding now.
+                    *c = false;
+                }
+
+                let utxo = &optional_utxos[current_selection.len() - 1];
+                curr_value -= utxo.effective_value as u64;
+            } else {
+                // Moving forwards, continuing down this branch
+                let utxo = &optional_utxos[current_selection.len()];
+
+                // Remove this utxo from the curr_available_value utxo amount
+                curr_available_value -= utxo.effective_value as u64;
+
+                // Inclusion branch first (Largest First Exploration)
+                current_selection.push(true);
+                curr_value += utxo.effective_value as u64;
+            }
+        }
+
+        // Check for solution
+        if best_selection.is_empty() {
+            return Err(Error::BnBTotalTriesExceeded);
+        }
+
+        // Set output set
+        let selected_utxos = optional_utxos
+            .into_iter()
+            .zip(best_selection)
+            .filter_map(|(optional, is_in_best)| if is_in_best { Some(optional) } else { None })
+            .collect();
+
+        Ok(BranchAndBoundCoinSelection::calculate_cs_result(
+            selected_utxos,
+            required_utxos,
+            fee_amount,
+        ))
+    }
+
+    fn single_random_draw(
+        &self,
+        required_utxos: Vec<OutputGroup>,
+        mut optional_utxos: Vec<OutputGroup>,
+        curr_value: u64,
+        actual_target: u64,
+        fee_amount: f32,
+    ) -> CoinSelectionResult {
+        #[cfg(not(test))]
+        optional_utxos.shuffle(&mut thread_rng());
+        #[cfg(test)]
+        {
+            let seed = [0; 32];
+            let mut rng: StdRng = SeedableRng::from_seed(seed);
+            optional_utxos.shuffle(&mut rng);
+        }
+
+        let selected_utxos = optional_utxos
+            .into_iter()
+            .scan(curr_value, |curr_value, utxo| {
+                if *curr_value >= actual_target {
+                    None
+                } else {
+                    *curr_value += utxo.effective_value as u64;
+                    Some(utxo)
+                }
+            })
+            .collect::<Vec<_>>();
+
+        BranchAndBoundCoinSelection::calculate_cs_result(selected_utxos, required_utxos, fee_amount)
+    }
+
+    fn calculate_cs_result(
+        mut selected_utxos: Vec<OutputGroup>,
+        mut required_utxos: Vec<OutputGroup>,
+        mut fee_amount: f32,
+    ) -> CoinSelectionResult {
+        selected_utxos.append(&mut required_utxos);
+        fee_amount += selected_utxos.iter().map(|u| u.fee).sum::<f32>();
+        let selected = selected_utxos
+            .into_iter()
+            .map(|u| u.utxo)
+            .collect::<Vec<_>>();
+        let selected_amount = selected.iter().map(|u| u.txout.value).sum();
+
+        CoinSelectionResult {
+            selected,
+            fee_amount,
+            selected_amount,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+
+    use bitcoin::{OutPoint, Script, TxOut};
+
+    use super::*;
+    use crate::database::MemoryDatabase;
+    use crate::types::*;
+
+    use rand::rngs::StdRng;
+    use rand::seq::SliceRandom;
+    use rand::{Rng, SeedableRng};
+
+    const P2WPKH_WITNESS_SIZE: usize = 73 + 33 + 2;
+
+    fn get_test_utxos() -> Vec<(UTXO, usize)> {
+        vec![
+            (
+                UTXO {
+                    outpoint: OutPoint::from_str(
+                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
+                    )
+                    .unwrap(),
+                    txout: TxOut {
+                        value: 100_000,
+                        script_pubkey: Script::new(),
+                    },
+                    keychain: KeychainKind::External,
+                },
+                P2WPKH_WITNESS_SIZE,
+            ),
+            (
+                UTXO {
+                    outpoint: OutPoint::from_str(
+                        "65d92ddff6b6dc72c89624a6491997714b90f6004f928d875bc0fd53f264fa85:0",
+                    )
+                    .unwrap(),
+                    txout: TxOut {
+                        value: 200_000,
+                        script_pubkey: Script::new(),
+                    },
+                    keychain: KeychainKind::Internal,
+                },
+                P2WPKH_WITNESS_SIZE,
+            ),
+        ]
+    }
+
+    fn generate_random_utxos(rng: &mut StdRng, utxos_number: usize) -> Vec<(UTXO, usize)> {
+        let mut res = Vec::new();
+        for _ in 0..utxos_number {
+            res.push((
+                UTXO {
+                    outpoint: OutPoint::from_str(
+                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
+                    )
+                    .unwrap(),
+                    txout: TxOut {
+                        value: rng.gen_range(0, 200000000),
+                        script_pubkey: Script::new(),
+                    },
+                    keychain: KeychainKind::External,
+                },
+                P2WPKH_WITNESS_SIZE,
+            ));
+        }
+        res
+    }
+
+    fn generate_same_value_utxos(utxos_value: u64, utxos_number: usize) -> Vec<(UTXO, usize)> {
+        let utxo = (
+            UTXO {
+                outpoint: OutPoint::from_str(
+                    "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
+                )
+                .unwrap(),
+                txout: TxOut {
+                    value: utxos_value,
+                    script_pubkey: Script::new(),
+                },
+                keychain: KeychainKind::External,
+            },
+            P2WPKH_WITNESS_SIZE,
+        );
+        vec![utxo; utxos_number]
+    }
+
+    fn sum_random_utxos(mut rng: &mut StdRng, utxos: &mut Vec<(UTXO, usize)>) -> u64 {
+        let utxos_picked_len = rng.gen_range(2, utxos.len() / 2);
+        utxos.shuffle(&mut rng);
+        utxos[..utxos_picked_len]
+            .iter()
+            .fold(0, |acc, x| acc + x.0.txout.value)
+    }
+
+    #[test]
+    fn test_largest_first_coin_selection_success() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let result = LargestFirstCoinSelection::default()
+            .coin_select(
+                &database,
+                utxos,
+                vec![],
+                FeeRate::from_sat_per_vb(1.0),
+                250_000,
+                50.0,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 2);
+        assert_eq!(result.selected_amount, 300_000);
+        assert_eq!(result.fee_amount, 186.0);
+    }
+
+    #[test]
+    fn test_largest_first_coin_selection_use_all() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let result = LargestFirstCoinSelection::default()
+            .coin_select(
+                &database,
+                utxos,
+                vec![],
+                FeeRate::from_sat_per_vb(1.0),
+                20_000,
+                50.0,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 2);
+        assert_eq!(result.selected_amount, 300_000);
+        assert_eq!(result.fee_amount, 186.0);
+    }
+
+    #[test]
+    fn test_largest_first_coin_selection_use_only_necessary() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let result = LargestFirstCoinSelection::default()
+            .coin_select(
+                &database,
+                vec![],
+                utxos,
+                FeeRate::from_sat_per_vb(1.0),
+                20_000,
+                50.0,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 1);
+        assert_eq!(result.selected_amount, 200_000);
+        assert_eq!(result.fee_amount, 118.0);
+    }
+
+    #[test]
+    #[should_panic(expected = "InsufficientFunds")]
+    fn test_largest_first_coin_selection_insufficient_funds() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        LargestFirstCoinSelection::default()
+            .coin_select(
+                &database,
+                vec![],
+                utxos,
+                FeeRate::from_sat_per_vb(1.0),
+                500_000,
+                50.0,
+            )
+            .unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "InsufficientFunds")]
+    fn test_largest_first_coin_selection_insufficient_funds_high_fees() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        LargestFirstCoinSelection::default()
+            .coin_select(
+                &database,
+                vec![],
+                utxos,
+                FeeRate::from_sat_per_vb(1000.0),
+                250_000,
+                50.0,
+            )
+            .unwrap();
+    }
+
+    #[test]
+    fn test_bnb_coin_selection_success() {
+        // In this case bnb won't find a suitable match and single random draw will
+        // select three outputs
+        let utxos = generate_same_value_utxos(100_000, 20);
+
+        let database = MemoryDatabase::default();
+
+        let result = BranchAndBoundCoinSelection::default()
+            .coin_select(
+                &database,
+                vec![],
+                utxos,
+                FeeRate::from_sat_per_vb(1.0),
+                250_000,
+                50.0,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 3);
+        assert_eq!(result.selected_amount, 300_000);
+        assert_eq!(result.fee_amount, 254.0);
+    }
+
+    #[test]
+    fn test_bnb_coin_selection_required_are_enough() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let result = BranchAndBoundCoinSelection::default()
+            .coin_select(
+                &database,
+                utxos.clone(),
+                utxos,
+                FeeRate::from_sat_per_vb(1.0),
+                20_000,
+                50.0,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 2);
+        assert_eq!(result.selected_amount, 300_000);
+        assert_eq!(result.fee_amount, 186.0);
+    }
+
+    #[test]
+    #[should_panic(expected = "InsufficientFunds")]
+    fn test_bnb_coin_selection_insufficient_funds() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        BranchAndBoundCoinSelection::default()
+            .coin_select(
+                &database,
+                vec![],
+                utxos,
+                FeeRate::from_sat_per_vb(1.0),
+                500_000,
+                50.0,
+            )
+            .unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "InsufficientFunds")]
+    fn test_bnb_coin_selection_insufficient_funds_high_fees() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        BranchAndBoundCoinSelection::default()
+            .coin_select(
+                &database,
+                vec![],
+                utxos,
+                FeeRate::from_sat_per_vb(1000.0),
+                250_000,
+                50.0,
+            )
+            .unwrap();
+    }
+
+    #[test]
+    fn test_bnb_coin_selection_check_fee_rate() {
+        let utxos = get_test_utxos();
+        let database = MemoryDatabase::default();
+
+        let result = BranchAndBoundCoinSelection::new(0)
+            .coin_select(
+                &database,
+                vec![],
+                utxos.clone(),
+                FeeRate::from_sat_per_vb(1.0),
+                99932, // first utxo's effective value
+                0.0,
+            )
+            .unwrap();
+
+        assert_eq!(result.selected.len(), 1);
+        assert_eq!(result.selected_amount, 100_000);
+        let input_size = (TXIN_BASE_WEIGHT as f32) / 4.0 + P2WPKH_WITNESS_SIZE as f32 / 4.0;
+        let epsilon = 0.5;
+        assert!((1.0 - (result.fee_amount / input_size)).abs() < epsilon);
+    }
+
+    #[test]
+    fn test_bnb_coin_selection_exact_match() {
+        let seed = [0; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let database = MemoryDatabase::default();
+
+        for _i in 0..200 {
+            let mut optional_utxos = generate_random_utxos(&mut rng, 16);
+            let target_amount = sum_random_utxos(&mut rng, &mut optional_utxos);
+            let result = BranchAndBoundCoinSelection::new(0)
+                .coin_select(
+                    &database,
+                    vec![],
+                    optional_utxos,
+                    FeeRate::from_sat_per_vb(0.0),
+                    target_amount,
+                    0.0,
+                )
+                .unwrap();
+            assert_eq!(result.selected_amount, target_amount);
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "BnBNoExactMatch")]
+    fn test_bnb_function_no_exact_match() {
+        let fee_rate = FeeRate::from_sat_per_vb(10.0);
+        let utxos: Vec<OutputGroup> = get_test_utxos()
+            .into_iter()
+            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .collect();
+
+        let curr_available_value = utxos
+            .iter()
+            .fold(0, |acc, x| acc + x.effective_value as u64);
+
+        let size_of_change = 31;
+        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_vb();
+        BranchAndBoundCoinSelection::new(size_of_change)
+            .bnb(
+                vec![],
+                utxos,
+                0,
+                curr_available_value,
+                20_000,
+                50.0,
+                cost_of_change,
+            )
+            .unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "BnBTotalTriesExceeded")]
+    fn test_bnb_function_tries_exceeded() {
+        let fee_rate = FeeRate::from_sat_per_vb(10.0);
+        let utxos: Vec<OutputGroup> = generate_same_value_utxos(100_000, 100_000)
+            .into_iter()
+            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .collect();
+
+        let curr_available_value = utxos
+            .iter()
+            .fold(0, |acc, x| acc + x.effective_value as u64);
+
+        let size_of_change = 31;
+        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_vb();
+
+        BranchAndBoundCoinSelection::new(size_of_change)
+            .bnb(
+                vec![],
+                utxos,
+                0,
+                curr_available_value,
+                20_000,
+                50.0,
+                cost_of_change,
+            )
+            .unwrap();
+    }
+
+    // The match won't be exact but still in the range
+    #[test]
+    fn test_bnb_function_almost_exact_match_with_fees() {
+        let fee_rate = FeeRate::from_sat_per_vb(1.0);
+        let size_of_change = 31;
+        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_vb();
+        let fee_amount = 50.0;
+
+        let utxos: Vec<_> = generate_same_value_utxos(50_000, 10)
+            .into_iter()
+            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .collect();
+
+        let curr_value = 0;
+
+        let curr_available_value = utxos
+            .iter()
+            .fold(0, |acc, x| acc + x.effective_value as u64);
+
+        // 2*(value of 1 utxo)  - 2*(1 utxo fees with 1.0sat/vbyte fee rate) -
+        // cost_of_change + 5.
+        let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change.ceil() as u64 + 5;
+
+        let result = BranchAndBoundCoinSelection::new(size_of_change)
+            .bnb(
+                vec![],
+                utxos,
+                curr_value,
+                curr_available_value,
+                target_amount,
+                fee_amount,
+                cost_of_change,
+            )
+            .unwrap();
+        assert_eq!(result.fee_amount, 186.0);
+        assert_eq!(result.selected_amount, 100_000);
+    }
+
+    // TODO: bnb() function should be optimized, and this test should be done with more utxos
+    #[test]
+    fn test_bnb_function_exact_match_more_utxos() {
+        let seed = [0; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let fee_rate = FeeRate::from_sat_per_vb(0.0);
+
+        for _ in 0..200 {
+            let optional_utxos: Vec<_> = generate_random_utxos(&mut rng, 40)
+                .into_iter()
+                .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+                .collect();
+
+            let curr_value = 0;
+
+            let curr_available_value = optional_utxos
+                .iter()
+                .fold(0, |acc, x| acc + x.effective_value as u64);
+
+            let target_amount = optional_utxos[3].effective_value as u64
+                + optional_utxos[23].effective_value as u64;
+
+            let result = BranchAndBoundCoinSelection::new(0)
+                .bnb(
+                    vec![],
+                    optional_utxos,
+                    curr_value,
+                    curr_available_value,
+                    target_amount,
+                    0.0,
+                    0.0,
+                )
+                .unwrap();
+            assert_eq!(result.selected_amount, target_amount);
+        }
+    }
+
+    #[test]
+    fn test_single_random_draw_function_success() {
+        let seed = [0; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let mut utxos = generate_random_utxos(&mut rng, 300);
+        let target_amount = sum_random_utxos(&mut rng, &mut utxos);
+
+        let fee_rate = FeeRate::from_sat_per_vb(1.0);
+        let utxos: Vec<OutputGroup> = utxos
+            .into_iter()
+            .map(|u| OutputGroup::new(u.0, u.1, fee_rate))
+            .collect();
+
+        let result = BranchAndBoundCoinSelection::default().single_random_draw(
+            vec![],
+            utxos,
+            0,
+            target_amount,
+            50.0,
+        );
+
+        assert!(result.selected_amount > target_amount);
+        assert_eq!(
+            result.fee_amount,
+            50.0 + result.selected.len() as f32 * 68.0
+        );
+    }
+}

src/wallet/export.rs

@@ -0,0 +1,343 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Wallet export
+//!
+//! This modules implements the wallet export format used by [FullyNoded](https://github.com/Fonta1n3/FullyNoded/blob/10b7808c8b929b171cca537fb50522d015168ac9/Docs/Wallets/Wallet-Export-Spec.md).
+//!
+//! ## Examples
+//!
+//! ### Import from JSON
+//!
+//! ```
+//! # use std::str::FromStr;
+//! # use bitcoin::*;
+//! # use bdk::database::*;
+//! # use bdk::wallet::export::*;
+//! # use bdk::*;
+//! let import = r#"{
+//!     "descriptor": "wpkh([c258d2e4\/84h\/1h\/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe\/0\/*)",
+//!     "blockheight":1782088,
+//!     "label":"testnet"
+//! }"#;
+//!
+//! let import = WalletExport::from_str(import)?;
+//! let wallet: OfflineWallet<_> = Wallet::new_offline(
+//!     &import.descriptor(),
+//!     import.change_descriptor().as_ref(),
+//!     Network::Testnet,
+//!     MemoryDatabase::default(),
+//! )?;
+//! # Ok::<_, bdk::Error>(())
+//! ```
+//!
+//! ### Export a `Wallet`
+//! ```
+//! # use bitcoin::*;
+//! # use bdk::database::*;
+//! # use bdk::wallet::export::*;
+//! # use bdk::*;
+//! let wallet: OfflineWallet<_> = Wallet::new_offline(
+//!     "wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/0/*)",
+//!     Some("wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/1/*)"),
+//!     Network::Testnet,
+//!     MemoryDatabase::default()
+//! )?;
+//! let export = WalletExport::export_wallet(&wallet, "exported wallet", true)
+//!     .map_err(ToString::to_string)
+//!     .map_err(bdk::Error::Generic)?;
+//!
+//! println!("Exported: {}", export.to_string());
+//! # Ok::<_, bdk::Error>(())
+//! ```
+
+use std::str::FromStr;
+
+use serde::{Deserialize, Serialize};
+
+use miniscript::{Descriptor, DescriptorPublicKey, ScriptContext, Terminal};
+
+use crate::blockchain::BlockchainMarker;
+use crate::database::BatchDatabase;
+use crate::wallet::Wallet;
+
+/// Structure that contains the export of a wallet
+///
+/// For a usage example see [this module](crate::wallet::export)'s documentation.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalletExport {
+    descriptor: String,
+    /// Earliest block to rescan when looking for the wallet's transactions
+    pub blockheight: u32,
+    /// Arbitrary label for the wallet
+    pub label: String,
+}
+
+impl ToString for WalletExport {
+    fn to_string(&self) -> String {
+        serde_json::to_string(self).unwrap()
+    }
+}
+
+impl FromStr for WalletExport {
+    type Err = serde_json::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        serde_json::from_str(s)
+    }
+}
+
+impl WalletExport {
+    /// Export a wallet
+    ///
+    /// This function returns an error if it determines that the `wallet`'s descriptor(s) are not
+    /// supported by Bitcoin Core or don't follow the standard derivation paths defined by BIP44
+    /// and others.
+    ///
+    /// If `include_blockheight` is `true`, this function will look into the `wallet`'s database
+    /// for the oldest transaction it knows and use that as the earliest block to rescan.
+    ///
+    /// If the database is empty or `include_blockheight` is false, the `blockheight` field
+    /// returned will be `0`.
+    pub fn export_wallet<B: BlockchainMarker, D: BatchDatabase>(
+        wallet: &Wallet<B, D>,
+        label: &str,
+        include_blockheight: bool,
+    ) -> Result<Self, &'static str> {
+        let descriptor = wallet
+            .descriptor
+            .to_string_with_secret(&wallet.signers.as_key_map(wallet.secp_ctx()));
+        Self::is_compatible_with_core(&descriptor)?;
+
+        let blockheight = match wallet.database.borrow().iter_txs(false) {
+            _ if !include_blockheight => 0,
+            Err(_) => 0,
+            Ok(txs) => {
+                let mut heights = txs
+                    .into_iter()
+                    .map(|tx| tx.height.unwrap_or(0))
+                    .collect::<Vec<_>>();
+                heights.sort_unstable();
+
+                *heights.last().unwrap_or(&0)
+            }
+        };
+
+        let export = WalletExport {
+            descriptor,
+            label: label.into(),
+            blockheight,
+        };
+
+        let desc_to_string = |d: &Descriptor<DescriptorPublicKey>| {
+            d.to_string_with_secret(&wallet.change_signers.as_key_map(wallet.secp_ctx()))
+        };
+        if export.change_descriptor() != wallet.change_descriptor.as_ref().map(desc_to_string) {
+            return Err("Incompatible change descriptor");
+        }
+
+        Ok(export)
+    }
+
+    fn is_compatible_with_core(descriptor: &str) -> Result<(), &'static str> {
+        fn check_ms<Ctx: ScriptContext>(
+            terminal: Terminal<String, Ctx>,
+        ) -> Result<(), &'static str> {
+            if let Terminal::Multi(_, _) = terminal {
+                Ok(())
+            } else {
+                Err("The descriptor contains operators not supported by Bitcoin Core")
+            }
+        }
+
+        match Descriptor::<String>::from_str(descriptor).map_err(|_| "Invalid descriptor")? {
+            Descriptor::Pk(_)
+            | Descriptor::Pkh(_)
+            | Descriptor::Wpkh(_)
+            | Descriptor::ShWpkh(_) => Ok(()),
+            Descriptor::Sh(ms) => check_ms(ms.node),
+            Descriptor::Wsh(ms) | Descriptor::ShWsh(ms) => check_ms(ms.node),
+            _ => Err("The descriptor is not compatible with Bitcoin Core"),
+        }
+    }
+
+    /// Return the external descriptor
+    pub fn descriptor(&self) -> String {
+        self.descriptor.clone()
+    }
+
+    /// Return the internal descriptor, if present
+    pub fn change_descriptor(&self) -> Option<String> {
+        let replaced = self.descriptor.replace("/0/*", "/1/*");
+
+        if replaced != self.descriptor {
+            Some(replaced)
+        } else {
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+
+    use bitcoin::{Network, Txid};
+
+    use super::*;
+    use crate::database::{memory::MemoryDatabase, BatchOperations};
+    use crate::types::TransactionDetails;
+    use crate::wallet::{OfflineWallet, Wallet};
+
+    fn get_test_db() -> MemoryDatabase {
+        let mut db = MemoryDatabase::new();
+        db.set_tx(&TransactionDetails {
+            transaction: None,
+            txid: Txid::from_str(
+                "4ddff1fa33af17f377f62b72357b43107c19110a8009b36fb832af505efed98a",
+            )
+            .unwrap(),
+            timestamp: 12345678,
+            received: 100_000,
+            sent: 0,
+            fees: 500,
+            height: Some(5000),
+        })
+        .unwrap();
+
+        db
+    }
+
+    #[test]
+    fn test_export_bip44() {
+        let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
+        let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
+
+        let wallet: OfflineWallet<_> = Wallet::new_offline(
+            descriptor,
+            Some(change_descriptor),
+            Network::Bitcoin,
+            get_test_db(),
+        )
+        .unwrap();
+        let export = WalletExport::export_wallet(&wallet, "Test Label", true).unwrap();
+
+        assert_eq!(export.descriptor(), descriptor);
+        assert_eq!(export.change_descriptor(), Some(change_descriptor.into()));
+        assert_eq!(export.blockheight, 5000);
+        assert_eq!(export.label, "Test Label");
+    }
+
+    #[test]
+    #[should_panic(expected = "Incompatible change descriptor")]
+    fn test_export_no_change() {
+        // This wallet explicitly doesn't have a change descriptor. It should be impossible to
+        // export, because exporting this kind of external descriptor normally implies the
+        // existence of an internal descriptor
+
+        let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
+
+        let wallet: OfflineWallet<_> =
+            Wallet::new_offline(descriptor, None, Network::Bitcoin, get_test_db()).unwrap();
+        WalletExport::export_wallet(&wallet, "Test Label", true).unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "Incompatible change descriptor")]
+    fn test_export_incompatible_change() {
+        // This wallet has a change descriptor, but the derivation path is not in the "standard"
+        // bip44/49/etc format
+
+        let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
+        let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/50'/0'/1/*)";
+
+        let wallet: OfflineWallet<_> = Wallet::new_offline(
+            descriptor,
+            Some(change_descriptor),
+            Network::Bitcoin,
+            get_test_db(),
+        )
+        .unwrap();
+        WalletExport::export_wallet(&wallet, "Test Label", true).unwrap();
+    }
+
+    #[test]
+    fn test_export_multi() {
+        let descriptor = "wsh(multi(2,\
+                                [73756c7f/48'/0'/0'/2']tpubDCKxNyM3bLgbEX13Mcd8mYxbVg9ajDkWXMh29hMWBurKfVmBfWAM96QVP3zaUcN51HvkZ3ar4VwP82kC8JZhhux8vFQoJintSpVBwpFvyU3/0/*,\
+                                [f9f62194/48'/0'/0'/2']tpubDDp3ZSH1yCwusRppH7zgSxq2t1VEUyXSeEp8E5aFS8m43MknUjiF1bSLo3CGWAxbDyhF1XowA5ukPzyJZjznYk3kYi6oe7QxtX2euvKWsk4/0/*,\
+                                [c98b1535/48'/0'/0'/2']tpubDCDi5W4sP6zSnzJeowy8rQDVhBdRARaPhK1axABi8V1661wEPeanpEXj4ZLAUEoikVtoWcyK26TKKJSecSfeKxwHCcRrge9k1ybuiL71z4a/0/*\
+                          ))";
+        let change_descriptor = "wsh(multi(2,\
+                                       [73756c7f/48'/0'/0'/2']tpubDCKxNyM3bLgbEX13Mcd8mYxbVg9ajDkWXMh29hMWBurKfVmBfWAM96QVP3zaUcN51HvkZ3ar4VwP82kC8JZhhux8vFQoJintSpVBwpFvyU3/1/*,\
+                                       [f9f62194/48'/0'/0'/2']tpubDDp3ZSH1yCwusRppH7zgSxq2t1VEUyXSeEp8E5aFS8m43MknUjiF1bSLo3CGWAxbDyhF1XowA5ukPzyJZjznYk3kYi6oe7QxtX2euvKWsk4/1/*,\
+                                       [c98b1535/48'/0'/0'/2']tpubDCDi5W4sP6zSnzJeowy8rQDVhBdRARaPhK1axABi8V1661wEPeanpEXj4ZLAUEoikVtoWcyK26TKKJSecSfeKxwHCcRrge9k1ybuiL71z4a/1/*\
+                                 ))";
+
+        let wallet: OfflineWallet<_> = Wallet::new_offline(
+            descriptor,
+            Some(change_descriptor),
+            Network::Testnet,
+            get_test_db(),
+        )
+        .unwrap();
+        let export = WalletExport::export_wallet(&wallet, "Test Label", true).unwrap();
+
+        assert_eq!(export.descriptor(), descriptor);
+        assert_eq!(export.change_descriptor(), Some(change_descriptor.into()));
+        assert_eq!(export.blockheight, 5000);
+        assert_eq!(export.label, "Test Label");
+    }
+
+    #[test]
+    fn test_export_to_json() {
+        let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
+        let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
+
+        let wallet: OfflineWallet<_> = Wallet::new_offline(
+            descriptor,
+            Some(change_descriptor),
+            Network::Bitcoin,
+            get_test_db(),
+        )
+        .unwrap();
+        let export = WalletExport::export_wallet(&wallet, "Test Label", true).unwrap();
+
+        assert_eq!(export.to_string(), "{\"descriptor\":\"wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44\'/0\'/0\'/0/*)\",\"blockheight\":5000,\"label\":\"Test Label\"}");
+    }
+
+    #[test]
+    fn test_export_from_json() {
+        let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
+        let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
+
+        let import_str = "{\"descriptor\":\"wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44\'/0\'/0\'/0/*)\",\"blockheight\":5000,\"label\":\"Test Label\"}";
+        let export = WalletExport::from_str(import_str).unwrap();
+
+        assert_eq!(export.descriptor(), descriptor);
+        assert_eq!(export.change_descriptor(), Some(change_descriptor.into()));
+        assert_eq!(export.blockheight, 5000);
+        assert_eq!(export.label, "Test Label");
+    }
+}

src/wallet/offline_stream.rs

@@ -1,52 +0,0 @@
-use std::io::{self, Error, ErrorKind, Read, Write};
-
-#[derive(Clone, Debug)]
-pub struct OfflineStream;
-
-impl Read for OfflineStream {
-    fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
-        Err(Error::new(
-            ErrorKind::NotConnected,
-            "Trying to read from an OfflineStream",
-        ))
-    }
-}
-
-impl Write for OfflineStream {
-    fn write(&mut self, _buf: &[u8]) -> io::Result<usize> {
-        Err(Error::new(
-            ErrorKind::NotConnected,
-            "Trying to read from an OfflineStream",
-        ))
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        Err(Error::new(
-            ErrorKind::NotConnected,
-            "Trying to read from an OfflineStream",
-        ))
-    }
-}
-
-// #[cfg(any(feature = "electrum", feature = "default"))]
-// use electrum_client::Client;
-//
-// #[cfg(any(feature = "electrum", feature = "default"))]
-// impl OfflineStream {
-//     fn new_client() -> {
-//         use std::io::bufreader;
-//
-//         let stream = OfflineStream{};
-//         let buf_reader = BufReader::new(stream.clone());
-//
-//         Client {
-//             stream,
-//             buf_reader,
-//             headers: VecDeque::new(),
-//             script_notifications: BTreeMap::new(),
-//
-//             #[cfg(feature = "debug-calls")]
-//             calls: 0,
-//         }
-//     }
-// }

src/wallet/signer.rs

@@ -0,0 +1,676 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Generalized signers
+//!
+//! This module provides the ability to add customized signers to a [`Wallet`](super::Wallet)
+//! through the [`Wallet::add_signer`](super::Wallet::add_signer) function.
+//!
+//! ```
+//! # use std::sync::Arc;
+//! # use std::str::FromStr;
+//! # use bitcoin::secp256k1::{Secp256k1, All};
+//! # use bitcoin::*;
+//! # use bitcoin::util::psbt;
+//! # use bitcoin::util::bip32::Fingerprint;
+//! # use bdk::signer::*;
+//! # use bdk::database::*;
+//! # use bdk::*;
+//! # #[derive(Debug)]
+//! # struct CustomHSM;
+//! # impl CustomHSM {
+//! #     fn sign_input(&self, _psbt: &mut psbt::PartiallySignedTransaction, _input: usize) -> Result<(), SignerError> {
+//! #         Ok(())
+//! #     }
+//! #     fn connect() -> Self {
+//! #         CustomHSM
+//! #     }
+//! # }
+//! #[derive(Debug)]
+//! struct CustomSigner {
+//!     device: CustomHSM,
+//! }
+//!
+//! impl CustomSigner {
+//!     fn connect() -> Self {
+//!         CustomSigner { device: CustomHSM::connect() }
+//!     }
+//! }
+//!
+//! impl Signer for CustomSigner {
+//!     fn sign(
+//!         &self,
+//!         psbt: &mut psbt::PartiallySignedTransaction,
+//!         input_index: Option<usize>,
+//!         _secp: &Secp256k1<All>,
+//!     ) -> Result<(), SignerError> {
+//!         let input_index = input_index.ok_or(SignerError::InputIndexOutOfRange)?;
+//!         self.device.sign_input(psbt, input_index)?;
+//!
+//!         Ok(())
+//!     }
+//!
+//!     fn sign_whole_tx(&self) -> bool {
+//!         false
+//!     }
+//! }
+//!
+//! let custom_signer = CustomSigner::connect();
+//!
+//! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)";
+//! let mut wallet: OfflineWallet<_> = Wallet::new_offline(descriptor, None, Network::Testnet, MemoryDatabase::default())?;
+//! wallet.add_signer(
+//!     KeychainKind::External,
+//!     Fingerprint::from_str("e30f11b8").unwrap().into(),
+//!     SignerOrdering(200),
+//!     Arc::new(custom_signer)
+//! );
+//!
+//! # Ok::<_, bdk::Error>(())
+//! ```
+
+use std::cmp::Ordering;
+use std::collections::BTreeMap;
+use std::fmt;
+use std::ops::Bound::Included;
+use std::sync::Arc;
+
+use bitcoin::blockdata::opcodes;
+use bitcoin::blockdata::script::Builder as ScriptBuilder;
+use bitcoin::hashes::{hash160, Hash};
+use bitcoin::secp256k1::{Message, Secp256k1};
+use bitcoin::util::bip32::{ExtendedPrivKey, Fingerprint};
+use bitcoin::util::{bip143, psbt};
+use bitcoin::{PrivateKey, Script, SigHash, SigHashType};
+
+use miniscript::descriptor::{DescriptorSecretKey, DescriptorSinglePriv, DescriptorXKey, KeyMap};
+use miniscript::{Legacy, MiniscriptKey, Segwitv0};
+
+use super::utils::SecpCtx;
+use crate::descriptor::XKeyUtils;
+
+/// Identifier of a signer in the `SignersContainers`. Used as a key to find the right signer among
+/// multiple of them
+#[derive(Debug, Clone, Ord, PartialOrd, PartialEq, Eq, Hash)]
+pub enum SignerId {
+    /// Bitcoin HASH160 (RIPEMD160 after SHA256) hash of an ECDSA public key
+    PkHash(hash160::Hash),
+    /// The fingerprint of a BIP32 extended key
+    Fingerprint(Fingerprint),
+}
+
+impl From<hash160::Hash> for SignerId {
+    fn from(hash: hash160::Hash) -> SignerId {
+        SignerId::PkHash(hash)
+    }
+}
+
+impl From<Fingerprint> for SignerId {
+    fn from(fing: Fingerprint) -> SignerId {
+        SignerId::Fingerprint(fing)
+    }
+}
+
+/// Signing error
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum SignerError {
+    /// The private key is missing for the required public key
+    MissingKey,
+    /// The private key in use has the right fingerprint but derives differently than expected
+    InvalidKey,
+    /// The user canceled the operation
+    UserCanceled,
+    /// Input index is out of range
+    InputIndexOutOfRange,
+    /// The `non_witness_utxo` field of the transaction is required to sign this input
+    MissingNonWitnessUtxo,
+    /// The `non_witness_utxo` specified is invalid
+    InvalidNonWitnessUtxo,
+    /// The `witness_utxo` field of the transaction is required to sign this input
+    MissingWitnessUtxo,
+    /// The `witness_script` field of the transaction is requied to sign this input
+    MissingWitnessScript,
+    /// The fingerprint and derivation path are missing from the psbt input
+    MissingHDKeypath,
+}
+
+impl fmt::Display for SignerError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for SignerError {}
+
+/// Trait for signers
+///
+/// This trait can be implemented to provide customized signers to the wallet. For an example see
+/// [`this module`](crate::wallet::signer)'s documentation.
+pub trait Signer: fmt::Debug + Send + Sync {
+    /// Sign a PSBT
+    ///
+    /// The `input_index` argument is only provided if the wallet doesn't declare to sign the whole
+    /// transaction in one go (see [`Signer::sign_whole_tx`]). Otherwise its value is `None` and
+    /// can be ignored.
+    fn sign(
+        &self,
+        psbt: &mut psbt::PartiallySignedTransaction,
+        input_index: Option<usize>,
+        secp: &SecpCtx,
+    ) -> Result<(), SignerError>;
+
+    /// Return whether or not the signer signs the whole transaction in one go instead of every
+    /// input individually
+    fn sign_whole_tx(&self) -> bool;
+
+    /// Return the secret key for the signer
+    ///
+    /// This is used internally to reconstruct the original descriptor that may contain secrets.
+    /// External signers that are meant to keep key isolated should just return `None` here (which
+    /// is the default for this method, if not overridden).
+    fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
+        None
+    }
+}
+
+impl Signer for DescriptorXKey<ExtendedPrivKey> {
+    fn sign(
+        &self,
+        psbt: &mut psbt::PartiallySignedTransaction,
+        input_index: Option<usize>,
+        secp: &SecpCtx,
+    ) -> Result<(), SignerError> {
+        let input_index = input_index.unwrap();
+        if input_index >= psbt.inputs.len() {
+            return Err(SignerError::InputIndexOutOfRange);
+        }
+
+        let (public_key, deriv_path) = match psbt.inputs[input_index]
+            .hd_keypaths
+            .iter()
+            .filter_map(|(pk, &(fingerprint, ref path))| {
+                if self.matches(&(fingerprint, path.clone()), &secp).is_some() {
+                    Some((pk, path))
+                } else {
+                    None
+                }
+            })
+            .next()
+        {
+            Some((pk, full_path)) => (pk, full_path.clone()),
+            None => return Ok(()),
+        };
+
+        let derived_key = self.xkey.derive_priv(&secp, &deriv_path).unwrap();
+        if &derived_key.private_key.public_key(&secp) != public_key {
+            Err(SignerError::InvalidKey)
+        } else {
+            derived_key.private_key.sign(psbt, Some(input_index), secp)
+        }
+    }
+
+    fn sign_whole_tx(&self) -> bool {
+        false
+    }
+
+    fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
+        Some(DescriptorSecretKey::XPrv(self.clone()))
+    }
+}
+
+impl Signer for PrivateKey {
+    fn sign(
+        &self,
+        psbt: &mut psbt::PartiallySignedTransaction,
+        input_index: Option<usize>,
+        secp: &SecpCtx,
+    ) -> Result<(), SignerError> {
+        let input_index = input_index.unwrap();
+        if input_index >= psbt.inputs.len() {
+            return Err(SignerError::InputIndexOutOfRange);
+        }
+
+        let pubkey = self.public_key(&secp);
+        if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
+            return Ok(());
+        }
+
+        // FIXME: use the presence of `witness_utxo` as an indication that we should make a bip143
+        // sig. Does this make sense? Should we add an extra argument to explicitly swith between
+        // these? The original idea was to declare sign() as sign<Ctx: ScriptContex>() and use Ctx,
+        // but that violates the rules for trait-objects, so we can't do it.
+        let (hash, sighash) = match psbt.inputs[input_index].witness_utxo {
+            Some(_) => Segwitv0::sighash(psbt, input_index)?,
+            None => Legacy::sighash(psbt, input_index)?,
+        };
+
+        let signature = secp.sign(
+            &Message::from_slice(&hash.into_inner()[..]).unwrap(),
+            &self.key,
+        );
+
+        let mut final_signature = Vec::with_capacity(75);
+        final_signature.extend_from_slice(&signature.serialize_der());
+        final_signature.push(sighash.as_u32() as u8);
+
+        psbt.inputs[input_index]
+            .partial_sigs
+            .insert(pubkey, final_signature);
+
+        Ok(())
+    }
+
+    fn sign_whole_tx(&self) -> bool {
+        false
+    }
+
+    fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
+        Some(DescriptorSecretKey::SinglePriv(DescriptorSinglePriv {
+            key: *self,
+            origin: None,
+        }))
+    }
+}
+
+/// Defines the order in which signers are called
+///
+/// The default value is `100`. Signers with an ordering above that will be called later,
+/// and they will thus see the partial signatures added to the transaction once they get to sign
+/// themselves.
+#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq)]
+pub struct SignerOrdering(pub usize);
+
+impl std::default::Default for SignerOrdering {
+    fn default() -> Self {
+        SignerOrdering(100)
+    }
+}
+
+#[derive(Debug, Clone)]
+struct SignersContainerKey {
+    id: SignerId,
+    ordering: SignerOrdering,
+}
+
+impl From<(SignerId, SignerOrdering)> for SignersContainerKey {
+    fn from(tuple: (SignerId, SignerOrdering)) -> Self {
+        SignersContainerKey {
+            id: tuple.0,
+            ordering: tuple.1,
+        }
+    }
+}
+
+/// Container for multiple signers
+#[derive(Debug, Default, Clone)]
+pub struct SignersContainer(BTreeMap<SignersContainerKey, Arc<dyn Signer>>);
+
+impl SignersContainer {
+    /// Create a map of public keys to secret keys
+    pub fn as_key_map(&self, secp: &SecpCtx) -> KeyMap {
+        self.0
+            .values()
+            .filter_map(|signer| signer.descriptor_secret_key())
+            .filter_map(|secret| secret.as_public(secp).ok().map(|public| (public, secret)))
+            .collect()
+    }
+}
+
+impl From<KeyMap> for SignersContainer {
+    fn from(keymap: KeyMap) -> SignersContainer {
+        let secp = Secp256k1::new();
+        let mut container = SignersContainer::new();
+
+        for (_, secret) in keymap {
+            match secret {
+                DescriptorSecretKey::SinglePriv(private_key) => container.add_external(
+                    SignerId::from(
+                        private_key
+                            .key
+                            .public_key(&Secp256k1::signing_only())
+                            .to_pubkeyhash(),
+                    ),
+                    SignerOrdering::default(),
+                    Arc::new(private_key.key),
+                ),
+                DescriptorSecretKey::XPrv(xprv) => container.add_external(
+                    SignerId::from(xprv.root_fingerprint(&secp)),
+                    SignerOrdering::default(),
+                    Arc::new(xprv),
+                ),
+            };
+        }
+
+        container
+    }
+}
+
+impl SignersContainer {
+    /// Default constructor
+    pub fn new() -> Self {
+        SignersContainer(Default::default())
+    }
+
+    /// Adds an external signer to the container for the specified id. Optionally returns the
+    /// signer that was previously in the container, if any
+    pub fn add_external(
+        &mut self,
+        id: SignerId,
+        ordering: SignerOrdering,
+        signer: Arc<dyn Signer>,
+    ) -> Option<Arc<dyn Signer>> {
+        self.0.insert((id, ordering).into(), signer)
+    }
+
+    /// Removes a signer from the container and returns it
+    pub fn remove(&mut self, id: SignerId, ordering: SignerOrdering) -> Option<Arc<dyn Signer>> {
+        self.0.remove(&(id, ordering).into())
+    }
+
+    /// Returns the list of identifiers of all the signers in the container
+    pub fn ids(&self) -> Vec<&SignerId> {
+        self.0
+            .keys()
+            .map(|SignersContainerKey { id, .. }| id)
+            .collect()
+    }
+
+    /// Returns the list of signers in the container, sorted by lowest to highest `ordering`
+    pub fn signers(&self) -> Vec<&Arc<dyn Signer>> {
+        self.0.values().collect()
+    }
+
+    /// Finds the signer with lowest ordering for a given id in the container.
+    pub fn find(&self, id: SignerId) -> Option<&Arc<dyn Signer>> {
+        self.0
+            .range((
+                Included(&(id.clone(), SignerOrdering(0)).into()),
+                Included(&(id.clone(), SignerOrdering(usize::MAX)).into()),
+            ))
+            .filter(|(k, _)| k.id == id)
+            .map(|(_, v)| v)
+            .next()
+    }
+}
+
+pub(crate) trait ComputeSighash {
+    fn sighash(
+        psbt: &psbt::PartiallySignedTransaction,
+        input_index: usize,
+    ) -> Result<(SigHash, SigHashType), SignerError>;
+}
+
+impl ComputeSighash for Legacy {
+    fn sighash(
+        psbt: &psbt::PartiallySignedTransaction,
+        input_index: usize,
+    ) -> Result<(SigHash, SigHashType), SignerError> {
+        if input_index >= psbt.inputs.len() {
+            return Err(SignerError::InputIndexOutOfRange);
+        }
+
+        let psbt_input = &psbt.inputs[input_index];
+        let tx_input = &psbt.global.unsigned_tx.input[input_index];
+
+        let sighash = psbt_input.sighash_type.unwrap_or(SigHashType::All);
+        let script = match psbt_input.redeem_script {
+            Some(ref redeem_script) => redeem_script.clone(),
+            None => {
+                let non_witness_utxo = psbt_input
+                    .non_witness_utxo
+                    .as_ref()
+                    .ok_or(SignerError::MissingNonWitnessUtxo)?;
+                let prev_out = non_witness_utxo
+                    .output
+                    .get(tx_input.previous_output.vout as usize)
+                    .ok_or(SignerError::InvalidNonWitnessUtxo)?;
+
+                prev_out.script_pubkey.clone()
+            }
+        };
+
+        Ok((
+            psbt.global
+                .unsigned_tx
+                .signature_hash(input_index, &script, sighash.as_u32()),
+            sighash,
+        ))
+    }
+}
+
+fn p2wpkh_script_code(script: &Script) -> Script {
+    ScriptBuilder::new()
+        .push_opcode(opcodes::all::OP_DUP)
+        .push_opcode(opcodes::all::OP_HASH160)
+        .push_slice(&script[2..])
+        .push_opcode(opcodes::all::OP_EQUALVERIFY)
+        .push_opcode(opcodes::all::OP_CHECKSIG)
+        .into_script()
+}
+
+impl ComputeSighash for Segwitv0 {
+    fn sighash(
+        psbt: &psbt::PartiallySignedTransaction,
+        input_index: usize,
+    ) -> Result<(SigHash, SigHashType), SignerError> {
+        if input_index >= psbt.inputs.len() {
+            return Err(SignerError::InputIndexOutOfRange);
+        }
+
+        let psbt_input = &psbt.inputs[input_index];
+
+        let sighash = psbt_input.sighash_type.unwrap_or(SigHashType::All);
+
+        let witness_utxo = psbt_input
+            .witness_utxo
+            .as_ref()
+            .ok_or(SignerError::MissingNonWitnessUtxo)?;
+        let value = witness_utxo.value;
+
+        let script = match psbt_input.witness_script {
+            Some(ref witness_script) => witness_script.clone(),
+            None => {
+                if witness_utxo.script_pubkey.is_v0_p2wpkh() {
+                    p2wpkh_script_code(&witness_utxo.script_pubkey)
+                } else if psbt_input
+                    .redeem_script
+                    .as_ref()
+                    .map(Script::is_v0_p2wpkh)
+                    .unwrap_or(false)
+                {
+                    p2wpkh_script_code(&psbt_input.redeem_script.as_ref().unwrap())
+                } else {
+                    return Err(SignerError::MissingWitnessScript);
+                }
+            }
+        };
+
+        Ok((
+            bip143::SigHashCache::new(&psbt.global.unsigned_tx).signature_hash(
+                input_index,
+                &script,
+                value,
+                sighash,
+            ),
+            sighash,
+        ))
+    }
+}
+
+impl PartialOrd for SignersContainerKey {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for SignersContainerKey {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.ordering
+            .cmp(&other.ordering)
+            .then(self.id.cmp(&other.id))
+    }
+}
+
+impl PartialEq for SignersContainerKey {
+    fn eq(&self, other: &Self) -> bool {
+        self.id == other.id && self.ordering == other.ordering
+    }
+}
+
+impl Eq for SignersContainerKey {}
+
+#[cfg(test)]
+mod signers_container_tests {
+    use super::*;
+    use crate::descriptor;
+    use crate::descriptor::ToWalletDescriptor;
+    use crate::keys::{DescriptorKey, ToDescriptorKey};
+    use bitcoin::secp256k1::All;
+    use bitcoin::util::bip32;
+    use bitcoin::util::psbt::PartiallySignedTransaction;
+    use bitcoin::Network;
+    use miniscript::ScriptContext;
+    use std::str::FromStr;
+
+    // Signers added with the same ordering (like `Ordering::default`) created from `KeyMap`
+    // should be preserved and not overwritten.
+    // This happens usually when a set of signers is created from a descriptor with private keys.
+    #[test]
+    fn signers_with_same_ordering() {
+        let (prvkey1, _, _) = setup_keys(TPRV0_STR);
+        let (prvkey2, _, _) = setup_keys(TPRV1_STR);
+        let desc = descriptor!(sh(multi(2, prvkey1, prvkey2))).unwrap();
+        let (_, keymap) = desc.to_wallet_descriptor(Network::Testnet).unwrap();
+
+        let signers = SignersContainer::from(keymap);
+        assert_eq!(signers.ids().len(), 2);
+
+        let signers = signers.signers();
+        assert_eq!(signers.len(), 2);
+    }
+
+    #[test]
+    fn signers_sorted_by_ordering() {
+        let mut signers = SignersContainer::new();
+        let signer1 = Arc::new(DummySigner);
+        let signer2 = Arc::new(DummySigner);
+        let signer3 = Arc::new(DummySigner);
+
+        signers.add_external(
+            SignerId::Fingerprint(b"cafe"[..].into()),
+            SignerOrdering(1),
+            signer1.clone(),
+        );
+        signers.add_external(
+            SignerId::Fingerprint(b"babe"[..].into()),
+            SignerOrdering(2),
+            signer2.clone(),
+        );
+        signers.add_external(
+            SignerId::Fingerprint(b"feed"[..].into()),
+            SignerOrdering(3),
+            signer3.clone(),
+        );
+
+        // Check that signers are sorted from lowest to highest ordering
+        let signers = signers.signers();
+        assert_eq!(Arc::as_ptr(signers[0]), Arc::as_ptr(&signer1));
+        assert_eq!(Arc::as_ptr(signers[1]), Arc::as_ptr(&signer2));
+        assert_eq!(Arc::as_ptr(signers[2]), Arc::as_ptr(&signer3));
+    }
+
+    #[test]
+    fn find_signer_by_id() {
+        let mut signers = SignersContainer::new();
+        let signer1: Arc<dyn Signer> = Arc::new(DummySigner);
+        let signer2: Arc<dyn Signer> = Arc::new(DummySigner);
+        let signer3: Arc<dyn Signer> = Arc::new(DummySigner);
+        let signer4: Arc<dyn Signer> = Arc::new(DummySigner);
+
+        let id1 = SignerId::Fingerprint(b"cafe"[..].into());
+        let id2 = SignerId::Fingerprint(b"babe"[..].into());
+        let id3 = SignerId::Fingerprint(b"feed"[..].into());
+        let id_nonexistent = SignerId::Fingerprint(b"fefe"[..].into());
+
+        signers.add_external(id1.clone(), SignerOrdering(1), signer1.clone());
+        signers.add_external(id2.clone(), SignerOrdering(2), signer2.clone());
+        signers.add_external(id3.clone(), SignerOrdering(3), signer3.clone());
+
+        assert!(
+            matches!(signers.find(id1), Some(signer) if Arc::as_ptr(&signer1) == Arc::as_ptr(signer))
+        );
+        assert!(
+            matches!(signers.find(id2), Some(signer) if Arc::as_ptr(&signer2) == Arc::as_ptr(signer))
+        );
+        assert!(
+            matches!(signers.find(id3.clone()), Some(signer) if Arc::as_ptr(&signer3) == Arc::as_ptr(signer))
+        );
+
+        // The `signer4` has the same ID as `signer3` but lower ordering.
+        // It should be found by `id3` instead of `signer3`.
+        signers.add_external(id3.clone(), SignerOrdering(2), signer4.clone());
+        assert!(
+            matches!(signers.find(id3), Some(signer) if Arc::as_ptr(&signer4) == Arc::as_ptr(signer))
+        );
+
+        // Can't find anything with ID that doesn't exist
+        assert!(matches!(signers.find(id_nonexistent), None));
+    }
+
+    #[derive(Debug)]
+    struct DummySigner;
+    impl Signer for DummySigner {
+        fn sign(
+            &self,
+            _psbt: &mut PartiallySignedTransaction,
+            _input_index: Option<usize>,
+            _secp: &SecpCtx,
+        ) -> Result<(), SignerError> {
+            Ok(())
+        }
+
+        fn sign_whole_tx(&self) -> bool {
+            true
+        }
+    }
+
+    const TPRV0_STR:&str = "tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf";
+    const TPRV1_STR:&str = "tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N";
+
+    const PATH: &str = "m/44'/1'/0'/0";
+
+    fn setup_keys<Ctx: ScriptContext>(
+        tprv: &str,
+    ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
+        let secp: Secp256k1<All> = Secp256k1::new();
+        let path = bip32::DerivationPath::from_str(PATH).unwrap();
+        let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
+        let tpub = bip32::ExtendedPubKey::from_private(&secp, &tprv);
+        let fingerprint = tprv.fingerprint(&secp);
+        let prvkey = (tprv, path.clone()).to_descriptor_key().unwrap();
+        let pubkey = (tpub, path).to_descriptor_key().unwrap();
+
+        (prvkey, pubkey, fingerprint)
+    }
+}

src/wallet/time.rs

@@ -0,0 +1,86 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Cross-platform time
+//!
+//! This module provides a function to get the current timestamp that works on all the platforms
+//! supported by the library.
+//!
+//! It can be useful to compare it with the timestamps found in
+//! [`TransactionDetails`](crate::types::TransactionDetails).
+
+use std::time::Duration;
+
+#[cfg(target_arch = "wasm32")]
+use js_sys::Date;
+#[cfg(not(target_arch = "wasm32"))]
+use std::time::{Instant as SystemInstant, SystemTime, UNIX_EPOCH};
+
+/// Return the current timestamp in seconds
+#[cfg(not(target_arch = "wasm32"))]
+pub fn get_timestamp() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs()
+}
+/// Return the current timestamp in seconds
+#[cfg(target_arch = "wasm32")]
+pub fn get_timestamp() -> u64 {
+    let millis = Date::now();
+
+    (millis / 1000.0) as u64
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) struct Instant(SystemInstant);
+#[cfg(target_arch = "wasm32")]
+pub(crate) struct Instant(Duration);
+
+impl Instant {
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn new() -> Self {
+        Instant(SystemInstant::now())
+    }
+    #[cfg(target_arch = "wasm32")]
+    pub fn new() -> Self {
+        let millis = Date::now();
+
+        let secs = millis / 1000.0;
+        let nanos = (millis % 1000.0) * 1e6;
+
+        Instant(Duration::new(secs as u64, nanos as u32))
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn elapsed(&self) -> Duration {
+        self.0.elapsed()
+    }
+    #[cfg(target_arch = "wasm32")]
+    pub fn elapsed(&self) -> Duration {
+        let now = Instant::new();
+
+        now.0.checked_sub(self.0).unwrap_or(Duration::new(0, 0))
+    }
+}

src/wallet/tx_builder.rs

@@ -0,0 +1,738 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//! Transaction builder
+//!
+//! ## Example
+//!
+//! ```
+//! # use std::str::FromStr;
+//! # use bitcoin::*;
+//! # use bdk::*;
+//! # use bdk::wallet::tx_builder::CreateTx;
+//! # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+//! // Create a transaction with one output to `to_address` of 50_000 satoshi, with a custom fee rate
+//! // of 5.0 satoshi/vbyte, only spending non-change outputs and with RBF signaling
+//! // enabled
+//! let builder = TxBuilder::with_recipients(vec![(to_address.script_pubkey(), 50_000)])
+//!     .fee_rate(FeeRate::from_sat_per_vb(5.0))
+//!     .do_not_spend_change()
+//!     .enable_rbf();
+//! # let builder: TxBuilder<bdk::database::MemoryDatabase, _, CreateTx> = builder;
+//! ```
+
+use std::collections::BTreeMap;
+use std::collections::HashSet;
+use std::default::Default;
+use std::marker::PhantomData;
+
+use bitcoin::{OutPoint, Script, SigHashType, Transaction};
+
+use super::coin_selection::{CoinSelectionAlgorithm, DefaultCoinSelectionAlgorithm};
+use crate::database::Database;
+use crate::types::{FeeRate, KeychainKind, UTXO};
+
+/// Context in which the [`TxBuilder`] is valid
+pub trait TxBuilderContext: std::fmt::Debug + Default + Clone {}
+
+/// [`Wallet::create_tx`](super::Wallet::create_tx) context
+#[derive(Debug, Default, Clone)]
+pub struct CreateTx;
+impl TxBuilderContext for CreateTx {}
+
+/// [`Wallet::bump_fee`](super::Wallet::bump_fee) context
+#[derive(Debug, Default, Clone)]
+pub struct BumpFee;
+impl TxBuilderContext for BumpFee {}
+
+/// A transaction builder
+///
+/// This structure contains the configuration that the wallet must follow to build a transaction.
+///
+/// For an example see [this module](super::tx_builder)'s documentation;
+#[derive(Debug)]
+pub struct TxBuilder<D: Database, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext> {
+    pub(crate) recipients: Vec<(Script, u64)>,
+    pub(crate) drain_wallet: bool,
+    pub(crate) single_recipient: Option<Script>,
+    pub(crate) fee_policy: Option<FeePolicy>,
+    pub(crate) internal_policy_path: Option<BTreeMap<String, Vec<usize>>>,
+    pub(crate) external_policy_path: Option<BTreeMap<String, Vec<usize>>>,
+    pub(crate) utxos: Vec<OutPoint>,
+    pub(crate) unspendable: HashSet<OutPoint>,
+    pub(crate) manually_selected_only: bool,
+    pub(crate) sighash: Option<SigHashType>,
+    pub(crate) ordering: TxOrdering,
+    pub(crate) locktime: Option<u32>,
+    pub(crate) rbf: Option<RBFValue>,
+    pub(crate) version: Option<Version>,
+    pub(crate) change_policy: ChangeSpendPolicy,
+    pub(crate) force_non_witness_utxo: bool,
+    pub(crate) add_global_xpubs: bool,
+    pub(crate) coin_selection: Cs,
+    pub(crate) include_output_redeem_witness_script: bool,
+
+    phantom: PhantomData<(D, Ctx)>,
+}
+
+#[derive(Debug)]
+pub(crate) enum FeePolicy {
+    FeeRate(FeeRate),
+    FeeAmount(u64),
+}
+
+impl std::default::Default for FeePolicy {
+    fn default() -> Self {
+        FeePolicy::FeeRate(FeeRate::default_min_relay_fee())
+    }
+}
+
+// Unfortunately derive doesn't work with `PhantomData`: https://github.com/rust-lang/rust/issues/26925
+impl<D: Database, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext> Default
+    for TxBuilder<D, Cs, Ctx>
+where
+    Cs: Default,
+{
+    fn default() -> Self {
+        TxBuilder {
+            recipients: Default::default(),
+            drain_wallet: Default::default(),
+            single_recipient: Default::default(),
+            fee_policy: Default::default(),
+            internal_policy_path: Default::default(),
+            external_policy_path: Default::default(),
+            utxos: Default::default(),
+            unspendable: Default::default(),
+            manually_selected_only: Default::default(),
+            sighash: Default::default(),
+            ordering: Default::default(),
+            locktime: Default::default(),
+            rbf: Default::default(),
+            version: Default::default(),
+            change_policy: Default::default(),
+            force_non_witness_utxo: Default::default(),
+            add_global_xpubs: Default::default(),
+            coin_selection: Default::default(),
+            include_output_redeem_witness_script: Default::default(),
+
+            phantom: PhantomData,
+        }
+    }
+}
+
+// methods supported by both contexts, but only for `DefaultCoinSelectionAlgorithm`
+impl<D: Database, Ctx: TxBuilderContext> TxBuilder<D, DefaultCoinSelectionAlgorithm, Ctx> {
+    /// Create an empty builder
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+// methods supported by both contexts, for any CoinSelectionAlgorithm
+impl<D: Database, Cs: CoinSelectionAlgorithm<D>, Ctx: TxBuilderContext> TxBuilder<D, Cs, Ctx> {
+    /// Set a custom fee rate
+    pub fn fee_rate(mut self, fee_rate: FeeRate) -> Self {
+        self.fee_policy = Some(FeePolicy::FeeRate(fee_rate));
+        self
+    }
+
+    /// Set an absolute fee
+    pub fn fee_absolute(mut self, fee_amount: u64) -> Self {
+        self.fee_policy = Some(FeePolicy::FeeAmount(fee_amount));
+        self
+    }
+
+    /// Set the policy path to use while creating the transaction for a given keychain.
+    ///
+    /// This method accepts a map where the key is the policy node id (see
+    /// [`Policy::id`](crate::descriptor::Policy::id)) and the value is the list of the indexes of
+    /// the items that are intended to be satisfied from the policy node (see
+    /// [`SatisfiableItem::Thresh::items`](crate::descriptor::policy::SatisfiableItem::Thresh::items)).
+    ///
+    /// ## Example
+    ///
+    /// An example of when the policy path is needed is the following descriptor:
+    /// `wsh(thresh(2,pk(A),sj:and_v(v:pk(B),n:older(6)),snj:and_v(v:pk(C),after(630000))))`,
+    /// derived from the miniscript policy `thresh(2,pk(A),and(pk(B),older(6)),and(pk(C),after(630000)))`.
+    /// It declares three descriptor fragments, and at the top level it uses `thresh()` to
+    /// ensure that at least two of them are satisfied. The individual fragments are:
+    ///
+    /// 1. `pk(A)`
+    /// 2. `and(pk(B),older(6))`
+    /// 3. `and(pk(C),after(630000))`
+    ///
+    /// When those conditions are combined in pairs, it's clear that the transaction needs to be created
+    /// differently depending on how the user intends to satisfy the policy afterwards:
+    ///
+    /// * If fragments `1` and `2` are used, the transaction will need to use a specific
+    ///   `n_sequence` in order to spend an `OP_CSV` branch.
+    /// * If fragments `1` and `3` are used, the transaction will need to use a specific `locktime`
+    ///   in order to spend an `OP_CLTV` branch.
+    /// * If fragments `2` and `3` are used, the transaction will need both.
+    ///
+    /// When the spending policy is represented as a tree (see
+    /// [`Wallet::policies`](super::Wallet::policies)), every node
+    /// is assigned a unique identifier that can be used in the policy path to specify which of
+    /// the node's children the user intends to satisfy: for instance, assuming the `thresh()`
+    /// root node of this example has an id of `aabbccdd`, the policy path map would look like:
+    ///
+    /// `{ "aabbccdd" => [0, 1] }`
+    ///
+    /// where the key is the node's id, and the value is a list of the children that should be
+    /// used, in no particular order.
+    ///
+    /// If a particularly complex descriptor has multiple ambiguous thresholds in its structure,
+    /// multiple entries can be added to the map, one for each node that requires an explicit path.
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// # use std::collections::BTreeMap;
+    /// # use bitcoin::*;
+    /// # use bdk::*;
+    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// let mut path = BTreeMap::new();
+    /// path.insert("aabbccdd".to_string(), vec![0, 1]);
+    ///
+    /// let builder = TxBuilder::with_recipients(vec![(to_address.script_pubkey(), 50_000)])
+    ///     .policy_path(path, KeychainKind::External);
+    /// # let builder: TxBuilder<bdk::database::MemoryDatabase, _, _> = builder;
+    /// ```
+    pub fn policy_path(
+        mut self,
+        policy_path: BTreeMap<String, Vec<usize>>,
+        keychain: KeychainKind,
+    ) -> Self {
+        let to_update = match keychain {
+            KeychainKind::Internal => &mut self.internal_policy_path,
+            KeychainKind::External => &mut self.external_policy_path,
+        };
+
+        *to_update = Some(policy_path);
+        self
+    }
+
+    /// Replace the internal list of utxos that **must** be spent with a new list
+    ///
+    /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
+    /// the "utxos" and the "unspendable" list, it will be spent.
+    pub fn utxos(mut self, utxos: Vec<OutPoint>) -> Self {
+        self.utxos = utxos;
+        self
+    }
+
+    /// Add a utxo to the internal list of utxos that **must** be spent
+    ///
+    /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
+    /// the "utxos" and the "unspendable" list, it will be spent.
+    pub fn add_utxo(mut self, utxo: OutPoint) -> Self {
+        self.utxos.push(utxo);
+        self
+    }
+
+    /// Only spend utxos added by [`add_utxo`] and [`utxos`].
+    ///
+    /// The wallet will **not** add additional utxos to the transaction even if they are needed to
+    /// make the transaction valid.
+    ///
+    /// [`add_utxo`]: Self::add_utxo
+    /// [`utxos`]: Self::utxos
+    pub fn manually_selected_only(mut self) -> Self {
+        self.manually_selected_only = true;
+        self
+    }
+
+    /// Replace the internal list of unspendable utxos with a new list
+    ///
+    /// It's important to note that the "must-be-spent" utxos added with [`TxBuilder::utxos`] and
+    /// [`TxBuilder::add_utxo`] have priority over these. See the docs of the two linked methods
+    /// for more details.
+    pub fn unspendable(mut self, unspendable: Vec<OutPoint>) -> Self {
+        self.unspendable = unspendable.into_iter().collect();
+        self
+    }
+
+    /// Add a utxo to the internal list of unspendable utxos
+    ///
+    /// It's important to note that the "must-be-spent" utxos added with [`TxBuilder::utxos`] and
+    /// [`TxBuilder::add_utxo`] have priority over this. See the docs of the two linked methods
+    /// for more details.
+    pub fn add_unspendable(mut self, unspendable: OutPoint) -> Self {
+        self.unspendable.insert(unspendable);
+        self
+    }
+
+    /// Sign with a specific sig hash
+    ///
+    /// **Use this option very carefully**
+    pub fn sighash(mut self, sighash: SigHashType) -> Self {
+        self.sighash = Some(sighash);
+        self
+    }
+
+    /// Choose the ordering for inputs and outputs of the transaction
+    pub fn ordering(mut self, ordering: TxOrdering) -> Self {
+        self.ordering = ordering;
+        self
+    }
+
+    /// Use a specific nLockTime while creating the transaction
+    ///
+    /// This can cause conflicts if the wallet's descriptors contain an "after" (OP_CLTV) operator.
+    pub fn nlocktime(mut self, locktime: u32) -> Self {
+        self.locktime = Some(locktime);
+        self
+    }
+
+    /// Build a transaction with a specific version
+    ///
+    /// The `version` should always be greater than `0` and greater than `1` if the wallet's
+    /// descriptors contain an "older" (OP_CSV) operator.
+    pub fn version(mut self, version: i32) -> Self {
+        self.version = Some(Version(version));
+        self
+    }
+
+    /// Do not spend change outputs
+    ///
+    /// This effectively adds all the change outputs to the "unspendable" list. See
+    /// [`TxBuilder::unspendable`].
+    pub fn do_not_spend_change(mut self) -> Self {
+        self.change_policy = ChangeSpendPolicy::ChangeForbidden;
+        self
+    }
+
+    /// Only spend change outputs
+    ///
+    /// This effectively adds all the non-change outputs to the "unspendable" list. See
+    /// [`TxBuilder::unspendable`].
+    pub fn only_spend_change(mut self) -> Self {
+        self.change_policy = ChangeSpendPolicy::OnlyChange;
+        self
+    }
+
+    /// Set a specific [`ChangeSpendPolicy`]. See [`TxBuilder::do_not_spend_change`] and
+    /// [`TxBuilder::only_spend_change`] for some shortcuts.
+    pub fn change_policy(mut self, change_policy: ChangeSpendPolicy) -> Self {
+        self.change_policy = change_policy;
+        self
+    }
+
+    /// Fill-in the [`psbt::Input::non_witness_utxo`](bitcoin::util::psbt::Input::non_witness_utxo) field even if the wallet only has SegWit
+    /// descriptors.
+    ///
+    /// This is useful for signers which always require it, like Trezor hardware wallets.
+    pub fn force_non_witness_utxo(mut self) -> Self {
+        self.force_non_witness_utxo = true;
+        self
+    }
+
+    /// Fill-in the [`psbt::Output::redeem_script`](bitcoin::util::psbt::Output::redeem_script) and
+    /// [`psbt::Output::witness_script`](bitcoin::util::psbt::Output::witness_script) fields.
+    ///
+    /// This is useful for signers which always require it, like ColdCard hardware wallets.
+    pub fn include_output_redeem_witness_script(mut self) -> Self {
+        self.include_output_redeem_witness_script = true;
+        self
+    }
+
+    /// Fill-in the `PSBT_GLOBAL_XPUB` field with the extended keys contained in both the external
+    /// and internal descriptors
+    ///
+    /// This is useful for offline signers that take part to a multisig. Some hardware wallets like
+    /// BitBox and ColdCard are known to require this.
+    pub fn add_global_xpubs(mut self) -> Self {
+        self.add_global_xpubs = true;
+        self
+    }
+
+    /// Spend all the available inputs. This respects filters like [`TxBuilder::unspendable`] and the change policy.
+    pub fn drain_wallet(mut self) -> Self {
+        self.drain_wallet = true;
+        self
+    }
+
+    /// Choose the coin selection algorithm
+    ///
+    /// Overrides the [`DefaultCoinSelectionAlgorithm`](super::coin_selection::DefaultCoinSelectionAlgorithm).
+    pub fn coin_selection<P: CoinSelectionAlgorithm<D>>(
+        self,
+        coin_selection: P,
+    ) -> TxBuilder<D, P, Ctx> {
+        TxBuilder {
+            recipients: self.recipients,
+            drain_wallet: self.drain_wallet,
+            single_recipient: self.single_recipient,
+            fee_policy: self.fee_policy,
+            internal_policy_path: self.internal_policy_path,
+            external_policy_path: self.external_policy_path,
+            utxos: self.utxos,
+            unspendable: self.unspendable,
+            manually_selected_only: self.manually_selected_only,
+            sighash: self.sighash,
+            ordering: self.ordering,
+            locktime: self.locktime,
+            rbf: self.rbf,
+            version: self.version,
+            change_policy: self.change_policy,
+            force_non_witness_utxo: self.force_non_witness_utxo,
+            add_global_xpubs: self.add_global_xpubs,
+            include_output_redeem_witness_script: self.include_output_redeem_witness_script,
+            coin_selection,
+
+            phantom: PhantomData,
+        }
+    }
+}
+
+// methods supported only by create_tx, and only for `DefaultCoinSelectionAlgorithm`
+impl<D: Database> TxBuilder<D, DefaultCoinSelectionAlgorithm, CreateTx> {
+    /// Create a builder starting from a list of recipients
+    pub fn with_recipients(recipients: Vec<(Script, u64)>) -> Self {
+        Self::default().set_recipients(recipients)
+    }
+}
+
+// methods supported only by create_tx, for any `CoinSelectionAlgorithm`
+impl<D: Database, Cs: CoinSelectionAlgorithm<D>> TxBuilder<D, Cs, CreateTx> {
+    /// Replace the recipients already added with a new list
+    pub fn set_recipients(mut self, recipients: Vec<(Script, u64)>) -> Self {
+        self.recipients = recipients;
+        self
+    }
+
+    /// Add a recipient to the internal list
+    pub fn add_recipient(mut self, script_pubkey: Script, amount: u64) -> Self {
+        self.recipients.push((script_pubkey, amount));
+        self
+    }
+
+    /// Set a single recipient that will get all the selected funds minus the fee. No change will
+    /// be created
+    ///
+    /// This method overrides any recipient set with [`set_recipients`](Self::set_recipients) or
+    /// [`add_recipient`](Self::add_recipient).
+    ///
+    /// It can only be used in conjunction with [`drain_wallet`](Self::drain_wallet) to send the
+    /// entire content of the wallet (minus filters) to a single recipient or with a
+    /// list of manually selected UTXOs by enabling [`manually_selected_only`](Self::manually_selected_only)
+    /// and selecting them with [`utxos`](Self::utxos) or [`add_utxo`](Self::add_utxo).
+    ///
+    /// When bumping the fees of a transaction made with this option, the user should remeber to
+    /// add [`maintain_single_recipient`](Self::maintain_single_recipient) to correctly update the
+    /// single output instead of adding one more for the change.
+    pub fn set_single_recipient(mut self, recipient: Script) -> Self {
+        self.single_recipient = Some(recipient);
+        self.recipients.clear();
+
+        self
+    }
+
+    /// Enable signaling RBF
+    ///
+    /// This will use the default nSequence value of `0xFFFFFFFD`.
+    pub fn enable_rbf(mut self) -> Self {
+        self.rbf = Some(RBFValue::Default);
+        self
+    }
+
+    /// Enable signaling RBF with a specific nSequence value
+    ///
+    /// This can cause conflicts if the wallet's descriptors contain an "older" (OP_CSV) operator
+    /// and the given `nsequence` is lower than the CSV value.
+    ///
+    /// If the `nsequence` is higher than `0xFFFFFFFD` an error will be thrown, since it would not
+    /// be a valid nSequence to signal RBF.
+    pub fn enable_rbf_with_sequence(mut self, nsequence: u32) -> Self {
+        self.rbf = Some(RBFValue::Value(nsequence));
+        self
+    }
+}
+
+// methods supported only by bump_fee
+impl<D: Database> TxBuilder<D, DefaultCoinSelectionAlgorithm, BumpFee> {
+    /// Bump the fees of a transaction made with [`set_single_recipient`](Self::set_single_recipient)
+    ///
+    /// Unless extra inputs are specified with [`add_utxo`] or [`utxos`], this flag will make
+    /// `bump_fee` reduce the value of the existing output, or fail if it would be consumed
+    /// entirely given the higher new fee rate.
+    ///
+    /// If extra inputs are added and they are not entirely consumed in fees, a change output will not
+    /// be added; the existing output will simply grow in value.
+    ///
+    /// Fails if the transaction has more than one outputs.
+    ///
+    /// [`add_utxo`]: Self::add_utxo
+    /// [`utxos`]: Self::utxos
+    pub fn maintain_single_recipient(mut self) -> Self {
+        self.single_recipient = Some(Script::default());
+        self
+    }
+}
+
+/// Ordering of the transaction's inputs and outputs
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+pub enum TxOrdering {
+    /// Randomized (default)
+    Shuffle,
+    /// Unchanged
+    Untouched,
+    /// BIP69 / Lexicographic
+    BIP69Lexicographic,
+}
+
+impl Default for TxOrdering {
+    fn default() -> Self {
+        TxOrdering::Shuffle
+    }
+}
+
+impl TxOrdering {
+    /// Sort transaction inputs and outputs by [`TxOrdering`] variant
+    pub fn sort_tx(&self, tx: &mut Transaction) {
+        match self {
+            TxOrdering::Untouched => {}
+            TxOrdering::Shuffle => {
+                use rand::seq::SliceRandom;
+                #[cfg(test)]
+                use rand::SeedableRng;
+
+                #[cfg(not(test))]
+                let mut rng = rand::thread_rng();
+                #[cfg(test)]
+                let mut rng = rand::rngs::StdRng::seed_from_u64(0);
+
+                tx.output.shuffle(&mut rng);
+            }
+            TxOrdering::BIP69Lexicographic => {
+                tx.input.sort_unstable_by_key(|txin| {
+                    (txin.previous_output.txid, txin.previous_output.vout)
+                });
+                tx.output
+                    .sort_unstable_by_key(|txout| (txout.value, txout.script_pubkey.clone()));
+            }
+        }
+    }
+}
+
+/// Transaction version
+///
+/// Has a default value of `1`
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+pub(crate) struct Version(pub(crate) i32);
+
+impl Default for Version {
+    fn default() -> Self {
+        Version(1)
+    }
+}
+
+/// RBF nSequence value
+///
+/// Has a default value of `0xFFFFFFFD`
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+pub(crate) enum RBFValue {
+    Default,
+    Value(u32),
+}
+
+impl RBFValue {
+    pub(crate) fn get_value(&self) -> u32 {
+        match self {
+            RBFValue::Default => 0xFFFFFFFD,
+            RBFValue::Value(v) => *v,
+        }
+    }
+}
+
+/// Policy regarding the use of change outputs when creating a transaction
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+pub enum ChangeSpendPolicy {
+    /// Use both change and non-change outputs (default)
+    ChangeAllowed,
+    /// Only use change outputs (see [`TxBuilder::only_spend_change`])
+    OnlyChange,
+    /// Only use non-change outputs (see [`TxBuilder::do_not_spend_change`])
+    ChangeForbidden,
+}
+
+impl Default for ChangeSpendPolicy {
+    fn default() -> Self {
+        ChangeSpendPolicy::ChangeAllowed
+    }
+}
+
+impl ChangeSpendPolicy {
+    pub(crate) fn is_satisfied_by(&self, utxo: &UTXO) -> bool {
+        match self {
+            ChangeSpendPolicy::ChangeAllowed => true,
+            ChangeSpendPolicy::OnlyChange => utxo.keychain == KeychainKind::Internal,
+            ChangeSpendPolicy::ChangeForbidden => utxo.keychain == KeychainKind::External,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    const ORDERING_TEST_TX: &'static str = "0200000003c26f3eb7932f7acddc5ddd26602b77e7516079b03090a16e2c2f54\
+                                            85d1fd600f0100000000ffffffffc26f3eb7932f7acddc5ddd26602b77e75160\
+                                            79b03090a16e2c2f5485d1fd600f0000000000ffffffff571fb3e02278217852\
+                                            dd5d299947e2b7354a639adc32ec1fa7b82cfb5dec530e0500000000ffffffff\
+                                            03e80300000000000002aaeee80300000000000001aa200300000000000001ff\
+                                            00000000";
+    macro_rules! ordering_test_tx {
+        () => {
+            deserialize::<bitcoin::Transaction>(&Vec::<u8>::from_hex(ORDERING_TEST_TX).unwrap())
+                .unwrap()
+        };
+    }
+
+    use bitcoin::consensus::deserialize;
+    use bitcoin::hashes::hex::FromHex;
+
+    use super::*;
+
+    #[test]
+    fn test_output_ordering_default_shuffle() {
+        assert_eq!(TxOrdering::default(), TxOrdering::Shuffle);
+    }
+
+    #[test]
+    fn test_output_ordering_untouched() {
+        let original_tx = ordering_test_tx!();
+        let mut tx = original_tx.clone();
+
+        TxOrdering::Untouched.sort_tx(&mut tx);
+
+        assert_eq!(original_tx, tx);
+    }
+
+    #[test]
+    fn test_output_ordering_shuffle() {
+        let original_tx = ordering_test_tx!();
+        let mut tx = original_tx.clone();
+
+        TxOrdering::Shuffle.sort_tx(&mut tx);
+
+        assert_eq!(original_tx.input, tx.input);
+        assert_ne!(original_tx.output, tx.output);
+    }
+
+    #[test]
+    fn test_output_ordering_bip69() {
+        use std::str::FromStr;
+
+        let original_tx = ordering_test_tx!();
+        let mut tx = original_tx.clone();
+
+        TxOrdering::BIP69Lexicographic.sort_tx(&mut tx);
+
+        assert_eq!(
+            tx.input[0].previous_output,
+            bitcoin::OutPoint::from_str(
+                "0e53ec5dfb2cb8a71fec32dc9a634a35b7e24799295ddd5278217822e0b31f57:5"
+            )
+            .unwrap()
+        );
+        assert_eq!(
+            tx.input[1].previous_output,
+            bitcoin::OutPoint::from_str(
+                "0f60fdd185542f2c6ea19030b0796051e7772b6026dd5ddccd7a2f93b73e6fc2:0"
+            )
+            .unwrap()
+        );
+        assert_eq!(
+            tx.input[2].previous_output,
+            bitcoin::OutPoint::from_str(
+                "0f60fdd185542f2c6ea19030b0796051e7772b6026dd5ddccd7a2f93b73e6fc2:1"
+            )
+            .unwrap()
+        );
+
+        assert_eq!(tx.output[0].value, 800);
+        assert_eq!(tx.output[1].script_pubkey, From::from(vec![0xAA]));
+        assert_eq!(tx.output[2].script_pubkey, From::from(vec![0xAA, 0xEE]));
+    }
+
+    fn get_test_utxos() -> Vec<UTXO> {
+        vec![
+            UTXO {
+                outpoint: OutPoint {
+                    txid: Default::default(),
+                    vout: 0,
+                },
+                txout: Default::default(),
+                keychain: KeychainKind::External,
+            },
+            UTXO {
+                outpoint: OutPoint {
+                    txid: Default::default(),
+                    vout: 1,
+                },
+                txout: Default::default(),
+                keychain: KeychainKind::Internal,
+            },
+        ]
+    }
+
+    #[test]
+    fn test_change_spend_policy_default() {
+        let change_spend_policy = ChangeSpendPolicy::default();
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
+
+        assert_eq!(filtered.len(), 2);
+    }
+
+    #[test]
+    fn test_change_spend_policy_no_internal() {
+        let change_spend_policy = ChangeSpendPolicy::ChangeForbidden;
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
+
+        assert_eq!(filtered.len(), 1);
+        assert_eq!(filtered[0].keychain, KeychainKind::External);
+    }
+
+    #[test]
+    fn test_change_spend_policy_only_internal() {
+        let change_spend_policy = ChangeSpendPolicy::OnlyChange;
+        let filtered = get_test_utxos()
+            .into_iter()
+            .filter(|u| change_spend_policy.is_satisfied_by(u))
+            .collect::<Vec<_>>();
+
+        assert_eq!(filtered.len(), 1);
+        assert_eq!(filtered[0].keychain, KeychainKind::Internal);
+    }
+
+    #[test]
+    fn test_default_tx_version_1() {
+        let version = Version::default();
+        assert_eq!(version.0, 1);
+    }
+}

src/wallet/utils.rs

@@ -1,10 +1,54 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+use bitcoin::secp256k1::{All, Secp256k1};
+use bitcoin::util::bip32;
+
+use miniscript::descriptor::DescriptorPublicKeyCtx;
+use miniscript::{MiniscriptKey, Satisfier, ToPublicKey};
+
 // De-facto standard "dust limit" (even though it should change based on the output type)
 const DUST_LIMIT_SATOSHI: u64 = 546;
 
+// MSB of the nSequence. If set there's no consensus-constraint, so it must be disabled when
+// spending using CSV in order to enforce CSV rules
+pub(crate) const SEQUENCE_LOCKTIME_DISABLE_FLAG: u32 = 1 << 31;
+// When nSequence is lower than this flag the timelock is interpreted as block-height-based,
+// otherwise it's time-based
+pub(crate) const SEQUENCE_LOCKTIME_TYPE_FLAG: u32 = 1 << 22;
+// Mask for the bits used to express the timelock
+pub(crate) const SEQUENCE_LOCKTIME_MASK: u32 = 0x0000FFFF;
+
+// Threshold for nLockTime to be considered a block-height-based timelock rather than time-based
+pub(crate) const BLOCKS_TIMELOCK_THRESHOLD: u32 = 500000000;
+
+/// Trait to check if a value is below the dust limit
 // we implement this trait to make sure we don't mess up the comparison with off-by-one like a <
 // instead of a <= etc. The constant value for the dust limit is not public on purpose, to
 // encourage the usage of this trait.
 pub trait IsDust {
+    /// Check whether or not a value is below dust limit
     fn is_dust(&self) -> bool;
 }
 
@@ -14,12 +58,114 @@ impl IsDust for u64 {
     }
 }
 
+pub struct After {
+    pub current_height: Option<u32>,
+    pub assume_height_reached: bool,
+}
+
+impl After {
+    pub(crate) fn new(current_height: Option<u32>, assume_height_reached: bool) -> After {
+        After {
+            current_height,
+            assume_height_reached,
+        }
+    }
+}
+
+pub(crate) fn check_nsequence_rbf(rbf: u32, csv: u32) -> bool {
+    // This flag cannot be set in the nSequence when spending using OP_CSV
+    if rbf & SEQUENCE_LOCKTIME_DISABLE_FLAG != 0 {
+        return false;
+    }
+
+    let mask = SEQUENCE_LOCKTIME_TYPE_FLAG | SEQUENCE_LOCKTIME_MASK;
+    let rbf = rbf & mask;
+    let csv = csv & mask;
+
+    // Both values should be represented in the same unit (either time-based or
+    // block-height based)
+    if (rbf < SEQUENCE_LOCKTIME_TYPE_FLAG) != (csv < SEQUENCE_LOCKTIME_TYPE_FLAG) {
+        return false;
+    }
+
+    // The value should be at least `csv`
+    if rbf < csv {
+        return false;
+    }
+
+    true
+}
+
+pub(crate) fn check_nlocktime(nlocktime: u32, required: u32) -> bool {
+    // Both values should be expressed in the same unit
+    if (nlocktime < BLOCKS_TIMELOCK_THRESHOLD) != (required < BLOCKS_TIMELOCK_THRESHOLD) {
+        return false;
+    }
+
+    // The value should be at least `required`
+    if nlocktime < required {
+        return false;
+    }
+
+    true
+}
+
+impl<ToPkCtx: Copy, Pk: MiniscriptKey + ToPublicKey<ToPkCtx>> Satisfier<ToPkCtx, Pk> for After {
+    fn check_after(&self, n: u32) -> bool {
+        if let Some(current_height) = self.current_height {
+            current_height >= n
+        } else {
+            self.assume_height_reached
+        }
+    }
+}
+
+pub struct Older {
+    pub current_height: Option<u32>,
+    pub create_height: Option<u32>,
+    pub assume_height_reached: bool,
+}
+
+impl Older {
+    pub(crate) fn new(
+        current_height: Option<u32>,
+        create_height: Option<u32>,
+        assume_height_reached: bool,
+    ) -> Older {
+        Older {
+            current_height,
+            create_height,
+            assume_height_reached,
+        }
+    }
+}
+
+impl<ToPkCtx: Copy, Pk: MiniscriptKey + ToPublicKey<ToPkCtx>> Satisfier<ToPkCtx, Pk> for Older {
+    fn check_older(&self, n: u32) -> bool {
+        if let Some(current_height) = self.current_height {
+            // TODO: test >= / >
+            current_height as u64 >= self.create_height.unwrap_or(0) as u64 + n as u64
+        } else {
+            self.assume_height_reached
+        }
+    }
+}
+
+pub(crate) type SecpCtx = Secp256k1<All>;
+pub(crate) fn descriptor_to_pk_ctx(secp: &SecpCtx) -> DescriptorPublicKeyCtx<'_, All> {
+    // Create a `to_pk_ctx` with a dummy derivation index, since we always use this on descriptor
+    // that have already been derived with `Descriptor::derive()`, so the child number added here
+    // is ignored.
+    DescriptorPublicKeyCtx::new(secp, bip32::ChildNumber::Normal { index: 0 })
+}
+
 pub struct ChunksIterator<I: Iterator> {
     iter: I,
     size: usize,
 }
 
 impl<I: Iterator> ChunksIterator<I> {
+    #[allow(dead_code)]
     pub fn new(iter: I, size: usize) -> Self {
         ChunksIterator { iter, size }
     }
@@ -46,3 +192,96 @@ impl<I: Iterator> Iterator for ChunksIterator<I> {
         Some(v)
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::{
+        check_nlocktime, check_nsequence_rbf, BLOCKS_TIMELOCK_THRESHOLD,
+        SEQUENCE_LOCKTIME_TYPE_FLAG,
+    };
+    use crate::types::FeeRate;
+
+    #[test]
+    fn test_fee_from_btc_per_kb() {
+        let fee = FeeRate::from_btc_per_kvb(1e-5);
+        assert!((fee.as_sat_vb() - 1.0).abs() < 0.0001);
+    }
+
+    #[test]
+    fn test_fee_from_sats_vbyte() {
+        let fee = FeeRate::from_sat_per_vb(1.0);
+        assert!((fee.as_sat_vb() - 1.0).abs() < 0.0001);
+    }
+
+    #[test]
+    fn test_fee_default_min_relay_fee() {
+        let fee = FeeRate::default_min_relay_fee();
+        assert!((fee.as_sat_vb() - 1.0).abs() < 0.0001);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_msb_set() {
+        let result = check_nsequence_rbf(0x80000000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_lt_csv() {
+        let result = check_nsequence_rbf(4000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_different_unit() {
+        let result = check_nsequence_rbf(SEQUENCE_LOCKTIME_TYPE_FLAG + 5000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_mask() {
+        let result = check_nsequence_rbf(0x3f + 10_000, 5000);
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_same_unit_blocks() {
+        let result = check_nsequence_rbf(10_000, 5000);
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nsequence_rbf_same_unit_time() {
+        let result = check_nsequence_rbf(
+            SEQUENCE_LOCKTIME_TYPE_FLAG + 10_000,
+            SEQUENCE_LOCKTIME_TYPE_FLAG + 5000,
+        );
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nlocktime_lt_cltv() {
+        let result = check_nlocktime(4000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nlocktime_different_unit() {
+        let result = check_nlocktime(BLOCKS_TIMELOCK_THRESHOLD + 5000, 5000);
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    fn test_check_nlocktime_same_unit_blocks() {
+        let result = check_nlocktime(10_000, 5000);
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_check_nlocktime_same_unit_time() {
+        let result = check_nlocktime(
+            BLOCKS_TIMELOCK_THRESHOLD + 10_000,
+            BLOCKS_TIMELOCK_THRESHOLD + 5000,
+        );
+        assert_eq!(result, true);
+    }
+}

testutils-macros/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "bdk-testutils-macros"
+version = "0.2.0"
+authors = ["Alekos Filini <alekos.filini@gmail.com>"]
+edition = "2018"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk-testutils-macros"
+description = "Supporting testing macros for `bdk`"
+keywords = ["bdk"]
+license-file = "../LICENSE"
+
+[lib]
+proc-macro = true
+name = "testutils_macros"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+syn = { version = "1.0", features = ["parsing", "full"] }
+proc-macro2 = "1.0"
+quote = "1.0"
+
+[features]
+debug = ["syn/extra-traits"]

testutils-macros/src/lib.rs

@@ -0,0 +1,545 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#[macro_use]
+extern crate quote;
+
+use proc_macro::TokenStream;
+
+use syn::spanned::Spanned;
+use syn::{parse, parse2, Ident, ReturnType};
+
+#[proc_macro_attribute]
+pub fn bdk_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let root_ident = if !attr.is_empty() {
+        match parse::<syn::ExprPath>(attr) {
+            Ok(parsed) => parsed,
+            Err(e) => {
+                let error_string = e.to_string();
+                return (quote! {
+                    compile_error!("Invalid crate path: {:?}", #error_string)
+                })
+                .into();
+            }
+        }
+    } else {
+        parse2::<syn::ExprPath>(quote! { bdk }).unwrap()
+    };
+
+    match parse::<syn::ItemFn>(item) {
+        Err(_) => (quote! {
+            compile_error!("#[bdk_blockchain_tests] can only be used on `fn`s")
+        })
+        .into(),
+        Ok(parsed) => {
+            let parsed_sig_ident = parsed.sig.ident.clone();
+            let mod_name = Ident::new(
+                &format!("generated_tests_{}", parsed_sig_ident.to_string()),
+                parsed.span(),
+            );
+
+            let return_type = match parsed.sig.output {
+                ReturnType::Type(_, ref t) => t.clone(),
+                ReturnType::Default => {
+                    return (quote! {
+                        compile_error!("The tagged function must return a type that impl `Blockchain`")
+                    }).into();
+                }
+            };
+
+            let output = quote! {
+
+            #parsed
+
+            mod #mod_name {
+                use bitcoin::Network;
+
+                use miniscript::Descriptor;
+
+                use testutils::{TestClient, serial};
+
+                use #root_ident::blockchain::{Blockchain, noop_progress};
+                use #root_ident::descriptor::ExtendedDescriptor;
+                use #root_ident::database::MemoryDatabase;
+                use #root_ident::types::KeychainKind;
+                use #root_ident::{Wallet, TxBuilder, FeeRate};
+
+                use super::*;
+
+                fn get_blockchain() -> #return_type {
+                    #parsed_sig_ident()
+                }
+
+                fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<#return_type, MemoryDatabase> {
+                    Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
+                }
+
+                fn init_single_sig() -> (Wallet<#return_type, MemoryDatabase>, (String, Option<String>), TestClient) {
+                    let descriptors = testutils! {
+                        @descriptors ( "wpkh(Alice)" ) ( "wpkh(Alice)" ) ( @keys ( "Alice" => (@generate_xprv "/44'/0'/0'/0/*", "/44'/0'/0'/1/*") ) )
+                    };
+
+                    let test_client = TestClient::new();
+                    let wallet = get_wallet_from_descriptors(&descriptors);
+
+                    (wallet, descriptors, test_client)
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_simple() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let tx = testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    };
+                    println!("{:?}", tx);
+                    let txid = test_client.receive(tx);
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.received, 50_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_stop_gap_20() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 5) => 50_000 )
+                    });
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 25) => 50_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 100_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_before_and_after_receive() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 0);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_multiple_outputs_same_tx() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000, (@external descriptors, 5) => 30_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 105_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 3);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.received, 105_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_receive_multi() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 5) => 25_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 2);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_address_reuse() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 25_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_receive_rbf_replaced() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 ) ( @replaceable true )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.received, 50_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+
+                    let new_txid = test_client.bump_fee(&txid);
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, new_txid);
+                    assert_eq!(list_tx_item.received, 50_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_reorg_block() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 ) ( @confirmations 1 ) ( @replaceable true )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert!(list_tx_item.height.is_some());
+
+                    // Invalidate 1 block
+                    test_client.invalidate(1);
+
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_after_send() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    println!("{}", descriptors.0);
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey(), 25_000)])).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    let tx = psbt.extract_tx();
+                    println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
+                    wallet.broadcast(tx).unwrap();
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), details.received);
+
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_outgoing_from_scratch() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    let received_txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey(), 25_000)])).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), details.received);
+
+                    // empty wallet
+                    let wallet = get_wallet_from_descriptors(&descriptors);
+                    wallet.sync(noop_progress(), None).unwrap();
+
+                    let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+
+                    let received = tx_map.get(&received_txid).unwrap();
+                    assert_eq!(received.received, 50_000);
+                    assert_eq!(received.sent, 0);
+
+                    let sent = tx_map.get(&sent_txid).unwrap();
+                    assert_eq!(sent.received, details.received);
+                    assert_eq!(sent.sent, details.sent);
+                    assert_eq!(sent.fees, details.fees);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_long_change_chain() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let mut total_sent = 0;
+                    for _ in 0..5 {
+                        let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 5_000)])).unwrap();
+                        let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                        assert!(finalized, "Cannot finalize transaction");
+                        wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                        wallet.sync(noop_progress(), None).unwrap();
+
+                        total_sent += 5_000 + details.fees;
+                    }
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+
+                    // empty wallet
+                    let wallet = get_wallet_from_descriptors(&descriptors);
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_bump_fee() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 5_000)]).enable_rbf()).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fees - 5_000);
+                    assert_eq!(wallet.get_balance().unwrap(), details.received);
+
+                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(2.1))).unwrap();
+                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fees - 5_000);
+                    assert_eq!(wallet.get_balance().unwrap(), new_details.received);
+
+                    assert!(new_details.fees > details.fees);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_bump_fee_remove_change() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 ) (@confirmations 1)
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 49_000)]).enable_rbf()).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fees);
+                    assert_eq!(wallet.get_balance().unwrap(), details.received);
+
+                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(5.0))).unwrap();
+
+                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 0);
+                    assert_eq!(new_details.received, 0);
+
+                    assert!(new_details.fees > details.fees);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_bump_fee_add_input() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 49_000)]).enable_rbf()).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
+                    assert_eq!(details.received, 1_000 - details.fees);
+
+                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(10.0))).unwrap();
+
+                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(new_details.sent, 75_000);
+                    assert_eq!(wallet.get_balance().unwrap(), new_details.received);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_bump_fee_add_input_no_change() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000 ) (@confirmations 1)
+                    });
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::with_recipients(vec![(node_addr.script_pubkey().clone(), 49_000)]).enable_rbf()).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees);
+                    assert_eq!(details.received, 1_000 - details.fees);
+
+                    let (new_psbt, new_details) = wallet.bump_fee(&details.txid, TxBuilder::new().fee_rate(FeeRate::from_sat_per_vb(123.0))).unwrap();
+                    println!("{:#?}", new_details);
+
+                    let (new_psbt, finalized) = wallet.sign(new_psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(new_psbt.extract_tx()).unwrap();
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(new_details.sent, 75_000);
+                    assert_eq!(wallet.get_balance().unwrap(), 0);
+                    assert_eq!(new_details.received, 0);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_receive_coinbase() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let wallet_addr = wallet.get_new_address().unwrap();
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 0);
+
+                    test_client.generate(1, Some(wallet_addr));
+
+                    wallet.sync(noop_progress(), None).unwrap();
+                    assert!(wallet.get_balance().unwrap() > 0);
+                }
+            }
+
+                        };
+
+            output.into()
+        }
+    }
+}

testutils/.gitignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

testutils/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "bdk-testutils"
+version = "0.2.0"
+authors = ["Alekos Filini <alekos.filini@gmail.com>"]
+edition = "2018"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk-testutils"
+description = "Supporting testing utilities for `bdk`"
+keywords = ["bdk"]
+license-file = "../LICENSE"
+
+[lib]
+name = "testutils"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4.8"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serial_test = "0.4"
+bitcoin = "0.25"
+bitcoincore-rpc = "0.12"
+electrum-client = "0.4.0-beta.1"

testutils/src/lib.rs

@@ -0,0 +1,532 @@
+// Magical Bitcoin Library
+// Written in 2020 by
+//     Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020 Magical Bitcoin
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#[macro_use]
+extern crate serde_json;
+
+pub use serial_test::serial;
+
+use std::collections::HashMap;
+use std::env;
+use std::ops::Deref;
+use std::path::PathBuf;
+use std::str::FromStr;
+use std::time::Duration;
+
+#[allow(unused_imports)]
+use log::{debug, error, info, trace};
+
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::sha256d;
+use bitcoin::{Address, Amount, Script, Transaction, Txid};
+
+pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
+pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
+
+pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
+
+// TODO: we currently only support env vars, we could also parse a toml file
+fn get_auth() -> Auth {
+    match env::var("MAGICAL_RPC_AUTH").as_ref().map(String::as_ref) {
+        Ok("USER_PASS") => Auth::UserPass(
+            env::var("MAGICAL_RPC_USER").unwrap(),
+            env::var("MAGICAL_RPC_PASS").unwrap(),
+        ),
+        _ => Auth::CookieFile(PathBuf::from(
+            env::var("MAGICAL_RPC_COOKIEFILE")
+                .unwrap_or("/home/user/.bitcoin/regtest/.cookie".to_string()),
+        )),
+    }
+}
+
+pub fn get_electrum_url() -> String {
+    env::var("MAGICAL_ELECTRUM_URL").unwrap_or("tcp://127.0.0.1:50001".to_string())
+}
+
+pub struct TestClient {
+    client: RpcClient,
+    electrum: ElectrumClient,
+}
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingOutput {
+    pub value: u64,
+    pub to_address: String,
+}
+
+impl TestIncomingOutput {
+    pub fn new(value: u64, to_address: Address) -> Self {
+        Self {
+            value,
+            to_address: to_address.to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingTx {
+    pub output: Vec<TestIncomingOutput>,
+    pub min_confirmations: Option<u64>,
+    pub locktime: Option<i64>,
+    pub replaceable: Option<bool>,
+}
+
+impl TestIncomingTx {
+    pub fn new(
+        output: Vec<TestIncomingOutput>,
+        min_confirmations: Option<u64>,
+        locktime: Option<i64>,
+        replaceable: Option<bool>,
+    ) -> Self {
+        Self {
+            output,
+            min_confirmations,
+            locktime,
+            replaceable,
+        }
+    }
+
+    pub fn add_output(&mut self, output: TestIncomingOutput) {
+        self.output.push(output);
+    }
+}
+
+#[macro_export]
+macro_rules! testutils {
+    ( @external $descriptors:expr, $child:expr ) => ({
+        use bitcoin::secp256k1::Secp256k1;
+        use miniscript::descriptor::{Descriptor, DescriptorPublicKey, DescriptorPublicKeyCtx};
+
+        let secp = Secp256k1::new();
+        let deriv_ctx = DescriptorPublicKeyCtx::new(&secp, bitcoin::util::bip32::ChildNumber::from_normal_idx(0).unwrap());
+
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&$descriptors.0).expect("Failed to parse descriptor in `testutils!(@external)`").0;
+        parsed.derive(bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()).address(bitcoin::Network::Regtest, deriv_ctx).expect("No address form")
+    });
+    ( @internal $descriptors:expr, $child:expr ) => ({
+        use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
+
+        let parsed = Descriptor::<DescriptorPublicKey>::parse_descriptor(&$descriptors.1.expect("Missing internal descriptor")).expect("Failed to parse descriptor in `testutils!(@internal)`").0;
+        parsed.derive(bitcoin::util::bip32::ChildNumber::from_normal_idx($child).unwrap()).address(bitcoin::Network::Regtest).expect("No address form")
+    });
+    ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
+    ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
+
+    ( @tx ( $( ( $( $addr:tt )* ) => $amount:expr ),+ ) $( ( @locktime $locktime:expr ) )* $( ( @confirmations $confirmations:expr ) )* $( ( @replaceable $replaceable:expr ) )* ) => ({
+        let mut outs = Vec::new();
+        $( outs.push(testutils::TestIncomingOutput::new($amount, testutils!( $($addr)* ))); )+
+
+        let mut locktime = None::<i64>;
+        $( locktime = Some($locktime); )*
+
+        let mut min_confirmations = None::<u64>;
+        $( min_confirmations = Some($confirmations); )*
+
+        let mut replaceable = None::<bool>;
+        $( replaceable = Some($replaceable); )*
+
+        testutils::TestIncomingTx::new(outs, min_confirmations, locktime, replaceable)
+    });
+
+    ( @literal $key:expr ) => ({
+        let key = $key.to_string();
+        (key, None::<String>, None::<String>)
+    });
+    ( @generate_xprv $( $external_path:expr )* $( ,$internal_path:expr )* ) => ({
+        use rand::Rng;
+
+        let mut seed = [0u8; 32];
+        rand::thread_rng().fill(&mut seed[..]);
+
+        let key = bitcoin::util::bip32::ExtendedPrivKey::new_master(
+            bitcoin::Network::Testnet,
+            &seed,
+        );
+
+        let mut external_path = None::<String>;
+        $( external_path = Some($external_path.to_string()); )*
+
+        let mut internal_path = None::<String>;
+        $( internal_path = Some($internal_path.to_string()); )*
+
+        (key.unwrap().to_string(), external_path, internal_path)
+    });
+    ( @generate_wif ) => ({
+        use rand::Rng;
+
+        let mut key = [0u8; bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
+        rand::thread_rng().fill(&mut key[..]);
+
+        (bitcoin::PrivateKey {
+            compressed: true,
+            network: bitcoin::Network::Testnet,
+            key: bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
+        }.to_string(), None::<String>, None::<String>)
+    });
+
+    ( @keys ( $( $alias:expr => ( $( $key_type:tt )* ) ),+ ) ) => ({
+        let mut map = std::collections::HashMap::new();
+        $(
+            let alias: &str = $alias;
+            map.insert(alias, testutils!( $($key_type)* ));
+        )+
+
+        map
+    });
+
+    ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )* $( ( @keys $( $keys:tt )* ) )* ) => ({
+        use std::str::FromStr;
+        use std::collections::HashMap;
+        use std::convert::TryInto;
+
+        use miniscript::descriptor::{Descriptor, DescriptorPublicKey};
+
+        let mut keys: HashMap<&'static str, (String, Option<String>, Option<String>)> = HashMap::new();
+        $(
+            keys = testutils!{ @keys $( $keys )* };
+        )*
+
+        let external: Descriptor<String> = FromStr::from_str($external_descriptor).unwrap();
+        let external: Descriptor<String> = external.translate_pk::<_, _, _, &'static str>(|k| {
+            if let Some((key, ext_path, _)) = keys.get(&k.as_str()) {
+                Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into())))
+            } else {
+                Ok(k.clone())
+            }
+        }, |kh| {
+            if let Some((key, ext_path, _)) = keys.get(&kh.as_str()) {
+                Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into())))
+            } else {
+                Ok(kh.clone())
+            }
+
+        }).unwrap();
+        let external = external.to_string();
+
+        let mut internal = None::<String>;
+        $(
+            let string_internal: Descriptor<String> = FromStr::from_str($internal_descriptor).unwrap();
+
+            let string_internal: Descriptor<String> = string_internal.translate_pk::<_, _, _, &'static str>(|k| {
+                if let Some((key, _, int_path)) = keys.get(&k.as_str()) {
+                    Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into())))
+                } else {
+                    Ok(k.clone())
+                }
+            }, |kh| {
+                if let Some((key, _, int_path)) = keys.get(&kh.as_str()) {
+                    Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into())))
+                } else {
+                    Ok(kh.clone())
+                }
+
+            }).unwrap();
+            internal = Some(string_internal.to_string());
+        )*
+
+        (external, internal)
+    })
+}
+
+fn exponential_backoff_poll<T, F>(mut poll: F) -> T
+where
+    F: FnMut() -> Option<T>,
+{
+    let mut delay = Duration::from_millis(64);
+    loop {
+        match poll() {
+            Some(data) => break data,
+            None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0),
+            None => {}
+        }
+
+        std::thread::sleep(delay);
+    }
+}
+
+impl TestClient {
+    pub fn new() -> Self {
+        let url = env::var("MAGICAL_RPC_URL").unwrap_or("127.0.0.1:18443".to_string());
+        let client = RpcClient::new(format!("http://{}", url), get_auth()).unwrap();
+        let electrum = ElectrumClient::new(&get_electrum_url()).unwrap();
+
+        TestClient { client, electrum }
+    }
+
+    fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
+        // wait for electrs to index the tx
+        exponential_backoff_poll(|| {
+            trace!("wait_for_tx {}", txid);
+
+            self.electrum
+                .script_get_history(monitor_script)
+                .unwrap()
+                .iter()
+                .position(|entry| entry.tx_hash == txid)
+        });
+    }
+
+    fn wait_for_block(&mut self, min_height: usize) {
+        self.electrum.block_headers_subscribe().unwrap();
+
+        loop {
+            let header = exponential_backoff_poll(|| {
+                self.electrum.ping().unwrap();
+                self.electrum.block_headers_pop().unwrap()
+            });
+            if header.height >= min_height {
+                break;
+            }
+        }
+    }
+
+    pub fn receive(&mut self, meta_tx: TestIncomingTx) -> Txid {
+        assert!(
+            meta_tx.output.len() > 0,
+            "can't create a transaction with no outputs"
+        );
+
+        let mut map = HashMap::new();
+
+        let mut required_balance = 0;
+        for out in &meta_tx.output {
+            required_balance += out.value;
+            map.insert(out.to_address.clone(), Amount::from_sat(out.value));
+        }
+
+        if self.get_balance(None, None).unwrap() < Amount::from_sat(required_balance) {
+            panic!("Insufficient funds in bitcoind. Plase generate a few blocks with: `bitcoin-cli generatetoaddress 10 {}`", self.get_new_address(None, None).unwrap());
+        }
+
+        // FIXME: core can't create a tx with two outputs to the same address
+        let tx = self
+            .create_raw_transaction_hex(&[], &map, meta_tx.locktime, meta_tx.replaceable)
+            .unwrap();
+        let tx = self.fund_raw_transaction(tx, None, None).unwrap();
+        let mut tx: Transaction = deserialize(&tx.hex).unwrap();
+
+        if let Some(true) = meta_tx.replaceable {
+            // for some reason core doesn't set this field right
+            for input in &mut tx.input {
+                input.sequence = 0xFFFFFFFD;
+            }
+        }
+
+        let tx = self
+            .sign_raw_transaction_with_wallet(&serialize(&tx), None, None)
+            .unwrap();
+
+        // broadcast through electrum so that it caches the tx immediately
+        let txid = self
+            .electrum
+            .transaction_broadcast(&deserialize(&tx.hex).unwrap())
+            .unwrap();
+
+        if let Some(num) = meta_tx.min_confirmations {
+            self.generate(num, None);
+        }
+
+        let monitor_script = Address::from_str(&meta_tx.output[0].to_address)
+            .unwrap()
+            .script_pubkey();
+        self.wait_for_tx(txid, &monitor_script);
+
+        debug!("Sent tx: {}", txid);
+
+        txid
+    }
+
+    pub fn bump_fee(&mut self, txid: &Txid) -> Txid {
+        let tx = self.get_raw_transaction_info(txid, None).unwrap();
+        assert!(
+            tx.confirmations.is_none(),
+            "Can't bump tx {} because it's already confirmed",
+            txid
+        );
+
+        let bumped: serde_json::Value = self.call("bumpfee", &[txid.to_string().into()]).unwrap();
+        let new_txid = Txid::from_str(&bumped["txid"].as_str().unwrap().to_string()).unwrap();
+
+        let monitor_script =
+            tx.vout[0].script_pub_key.addresses.as_ref().unwrap()[0].script_pubkey();
+        self.wait_for_tx(new_txid, &monitor_script);
+
+        debug!("Bumped {}, new txid {}", txid, new_txid);
+
+        new_txid
+    }
+
+    pub fn generate_manually(&mut self, txs: Vec<Transaction>) -> String {
+        use bitcoin::blockdata::block::{Block, BlockHeader};
+        use bitcoin::blockdata::script::Builder;
+        use bitcoin::blockdata::transaction::{OutPoint, TxIn, TxOut};
+        use bitcoin::hash_types::{BlockHash, TxMerkleNode};
+
+        let block_template: serde_json::Value = self
+            .call("getblocktemplate", &[json!({"rules": ["segwit"]})])
+            .unwrap();
+        trace!("getblocktemplate: {:#?}", block_template);
+
+        let header = BlockHeader {
+            version: block_template["version"].as_i64().unwrap() as i32,
+            prev_blockhash: BlockHash::from_hex(
+                block_template["previousblockhash"].as_str().unwrap(),
+            )
+            .unwrap(),
+            merkle_root: TxMerkleNode::default(),
+            time: block_template["curtime"].as_u64().unwrap() as u32,
+            bits: u32::from_str_radix(block_template["bits"].as_str().unwrap(), 16).unwrap(),
+            nonce: 0,
+        };
+        debug!("header: {:#?}", header);
+
+        let height = block_template["height"].as_u64().unwrap() as i64;
+        let witness_reserved_value: Vec<u8> = sha256d::Hash::default().as_ref().into();
+        // burn block subsidy and fees, not a big deal
+        let mut coinbase_tx = Transaction {
+            version: 1,
+            lock_time: 0,
+            input: vec![TxIn {
+                previous_output: OutPoint::null(),
+                script_sig: Builder::new().push_int(height).into_script(),
+                sequence: 0xFFFFFFFF,
+                witness: vec![witness_reserved_value],
+            }],
+            output: vec![],
+        };
+
+        let mut txdata = vec![coinbase_tx.clone()];
+        txdata.extend_from_slice(&txs);
+
+        let mut block = Block { header, txdata };
+
+        let witness_root = block.witness_root();
+        let witness_commitment =
+            Block::compute_witness_commitment(&witness_root, &coinbase_tx.input[0].witness[0]);
+
+        // now update and replace the coinbase tx
+        let mut coinbase_witness_commitment_script = vec![0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed];
+        coinbase_witness_commitment_script.extend_from_slice(&witness_commitment);
+
+        coinbase_tx.output.push(TxOut {
+            value: 0,
+            script_pubkey: coinbase_witness_commitment_script.into(),
+        });
+        block.txdata[0] = coinbase_tx;
+
+        // set merkle root
+        let merkle_root = block.merkle_root();
+        block.header.merkle_root = merkle_root;
+
+        assert!(block.check_merkle_root());
+        assert!(block.check_witness_commitment());
+
+        // now do PoW :)
+        let target = block.header.target();
+        while block.header.validate_pow(&target).is_err() {
+            block.header.nonce = block.header.nonce.checked_add(1).unwrap(); // panic if we run out of nonces
+        }
+
+        let block_hex: String = serialize(&block).to_hex();
+        debug!("generated block hex: {}", block_hex);
+
+        self.electrum.block_headers_subscribe().unwrap();
+
+        let submit_result: serde_json::Value =
+            self.call("submitblock", &[block_hex.into()]).unwrap();
+        debug!("submitblock: {:?}", submit_result);
+        assert!(
+            submit_result.is_null(),
+            "submitblock error: {:?}",
+            submit_result.as_str()
+        );
+
+        self.wait_for_block(height as usize);
+
+        block.header.block_hash().to_hex()
+    }
+
+    pub fn generate(&mut self, num_blocks: u64, address: Option<Address>) {
+        let address = address.unwrap_or_else(|| self.get_new_address(None, None).unwrap());
+        let hashes = self.generate_to_address(num_blocks, &address).unwrap();
+        let best_hash = hashes.last().unwrap();
+        let height = self.get_block_info(best_hash).unwrap().height;
+
+        self.wait_for_block(height);
+
+        debug!("Generated blocks to new height {}", height);
+    }
+
+    pub fn invalidate(&mut self, num_blocks: u64) {
+        self.electrum.block_headers_subscribe().unwrap();
+
+        let best_hash = self.get_best_block_hash().unwrap();
+        let initial_height = self.get_block_info(&best_hash).unwrap().height;
+
+        let mut to_invalidate = best_hash;
+        for i in 1..=num_blocks {
+            trace!(
+                "Invalidating block {}/{} ({})",
+                i,
+                num_blocks,
+                to_invalidate
+            );
+
+            self.invalidate_block(&to_invalidate).unwrap();
+            to_invalidate = self.get_best_block_hash().unwrap();
+        }
+
+        self.wait_for_block(initial_height - num_blocks as usize);
+
+        debug!(
+            "Invalidated {} blocks to new height of {}",
+            num_blocks,
+            initial_height - num_blocks as usize
+        );
+    }
+
+    pub fn reorg(&mut self, num_blocks: u64) {
+        self.invalidate(num_blocks);
+        self.generate(num_blocks, None);
+    }
+
+    pub fn get_node_address(&self, address_type: Option<AddressType>) -> Address {
+        Address::from_str(
+            &self
+                .get_new_address(None, address_type)
+                .unwrap()
+                .to_string(),
+        )
+        .unwrap()
+    }
+}
+
+impl Deref for TestClient {
+    type Target = RpcClient;
+
+    fn deref(&self) -> &Self::Target {
+        &self.client
+    }
+}