frost/bdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge bitcoindevkit/bdk#1380: Simplified EsploraExt API

Browse Source 
96a9aa6e63 feat(chain): refactor `merge_chains` (志宇)
2f22987c9e chore(chain): fix comment (志宇)
daf588f016 feat(chain): optimize `merge_chains` (志宇)
77d35954c1 feat(chain)!: rm `local_chain::Update` (志宇)
1269b0610e test(chain): fix incorrect test case (志宇)
72fe65b65f feat(esplora)!: simplify chain update logic (志宇)
eded1a7ea0 feat(chain): introduce `CheckPoint::insert` (志宇)
519cd75d23 test(esplora): move esplora tests into src files (志宇)
a6e613e6b9 test(esplora): add `test_finalize_chain_update` (志宇)
494d253493 feat(testenv): add `genesis_hash` method (志宇)
886d72e3d5 chore(chain)!: rm `missing_heights` and `missing_heights_from` methods (志宇)
bd62aa0fe1 feat(esplora)!: remove `EsploraExt::update_local_chain` (志宇)
1e99793983 feat(testenv): add `make_checkpoint_tip` (志宇)

Pull request description:

  Fixes #1354

  ### Description

  Built on top of both #1369 and #1373, we simplify the `EsploraExt` API by removing the `update_local_chain` method and having `full_scan` and `sync` update the local chain in the same call. The `full_scan` and `sync` methods now takes in an additional input (`local_tip`) which provides us with the view of the `LocalChain` before the update. These methods now return structs `FullScanUpdate` and `SyncUpdate`.

  The examples are updated to use this new API. `TxGraph::missing_heights` and `tx_graph::ChangeSet::missing_heights_from` are no longer needed, therefore they are removed.

  Additionally, we used this opportunity to simplify the logic which updates `LocalChain`. We got rid of the `local_chain::Update` struct (which contained the update `CheckPoint` tip and a `bool` which signaled whether we want to introduce blocks below point of agreement). It turns out we can use something like `CheckPoint::insert` so the chain source can craft an update based on the old tip. This way, we can make better use of `merge_chains`' optimization that compares the `Arc` pointers of the local and update chain (before we were crafting the update chain NOT based on top of the previous local chain). With this, we no longer need the `Update::introduce_older_block` field since the logic will naturally break when we reach a matching `Arc` pointer.

  ### Notes to the reviewers

  * Obtaining the `LocalChain`'s update now happens within `EsploraExt::full_scan` and `EsploraExt::sync`. Creating the `LocalChain` update is now split into two methods (`fetch_latest_blocks` and `chain_update`) that are called before and after fetching transactions and anchors.
  * We need to duplicate code for `bdk_esplora`. One for blocking and one for async.

  ### Changelog notice

  * Changed `EsploraExt` API so that sync only requires one round of fetching data. The `local_chain_update` method is removed and the `local_tip` parameter is added to the `full_scan` and `sync` methods.
  * Removed `TxGraph::missing_heights` and `tx_graph::ChangeSet::missing_heights_from` methods.
  * Introduced `CheckPoint::insert` which allows convenient checkpoint-insertion. This is intended for use by chain-sources when crafting an update.
  * Refactored `merge_chains` to also return the resultant `CheckPoint` tip.
  * Optimized the update `LocalChain` logic - use the update `CheckPoint` as the new `CheckPoint` tip when possible.

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

  #### New Features:

  * [x] I've added tests for the new feature
  * [x] I've added docs for the new feature

ACKs for top commit:
  LLFourn:
    ACK 96a9aa6e63

Tree-SHA512: 3d4f2eab08a1fe94eb578c594126e99679f72e231680b2edd4bfb018ba1d998ca123b07acb2d19c644d5887fc36b8e42badba91cd09853df421ded04de45bf69
This commit is contained in:
志宇
2024-04-22 17:44:44 +08:00
parent 9800f8d88e 96a9aa6e63
commit 8e73998cfa
19 changed files with 1609 additions and 1120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
example-crates/example_bitcoind_rpc_polling/src/main.rs
View File

@@ -188,10 +188,7 @@ fn main() -> anyhow::Result<()> {
let mut db = db.lock().unwrap();
let chain_changeset = chain
.apply_update(local_chain::Update {
tip: emission.checkpoint,
introduce_older_blocks: false,
})
.apply_update(emission.checkpoint)
.expect("must always apply as we receive blocks in order from emitter");
let graph_changeset = graph.apply_block_relevant(&emission.block, height);
db.stage((chain_changeset, graph_changeset));
@@ -301,12 +298,8 @@ fn main() -> anyhow::Result<()> {
let changeset = match emission {
Emission::Block(block_emission) => {
let height = block_emission.block_height();
let chain_update = local_chain::Update {
tip: block_emission.checkpoint,
introduce_older_blocks: false,
};
let chain_changeset = chain
.apply_update(chain_update)
.apply_update(block_emission.checkpoint)
.expect("must always apply as we receive blocks in order from emitter");
let graph_changeset =
graph.apply_block_relevant(&block_emission.block, height);

86
example-crates/example_esplora/src/main.rs
View File

@@ -1,5 +1,5 @@
use std::{
collections::{BTreeMap, BTreeSet},
collections::BTreeMap,
io::{self, Write},
sync::Mutex,
};
@@ -60,6 +60,7 @@ enum EsploraCommands {
esplora_args: EsploraArgs,
},
}
impl EsploraCommands {
fn esplora_args(&self) -> EsploraArgs {
match self {
@@ -149,20 +150,24 @@ fn main() -> anyhow::Result<()> {
};
let client = esplora_cmd.esplora_args().client(args.network)?;
// Prepare the `IndexedTxGraph` update based on whether we are scanning or syncing.
// Prepare the `IndexedTxGraph` and `LocalChain` updates based on whether we are scanning or
// syncing.
//
// Scanning: We are iterating through spks of all keychains and scanning for transactions for
// each spk. We start with the lowest derivation index spk and stop scanning after `stop_gap`
// number of consecutive spks have no transaction history. A Scan is done in situations of
// wallet restoration. It is a special case. Applications should use "sync" style updates
// after an initial scan.
//
// Syncing: We only check for specified spks, utxos and txids to update their confirmation
// status or fetch missing transactions.
let indexed_tx_graph_changeset = match &esplora_cmd {
let (local_chain_changeset, indexed_tx_graph_changeset) = match &esplora_cmd {
EsploraCommands::Scan {
stop_gap,
scan_options,
..
} => {
let local_tip = chain.lock().expect("mutex must not be poisoned").tip();
let keychain_spks = graph
.lock()
.expect("mutex must not be poisoned")
@@ -189,23 +194,33 @@ fn main() -> anyhow::Result<()> {
// is reached. It returns a `TxGraph` update (`graph_update`) and a structure that
// represents the last active spk derivation indices of keychains
// (`keychain_indices_update`).
let (mut graph_update, last_active_indices) = client
.full_scan(keychain_spks, *stop_gap, scan_options.parallel_requests)
let mut update = client
.full_scan(
local_tip,
keychain_spks,
*stop_gap,
scan_options.parallel_requests,
)
.context("scanning for transactions")?;
// We want to keep track of the latest time a transaction was seen unconfirmed.
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = graph_update.update_last_seen_unconfirmed(now);
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
let mut graph = graph.lock().expect("mutex must not be poisoned");
let mut chain = chain.lock().expect("mutex must not be poisoned");
// Because we did a stop gap based scan we are likely to have some updates to our
// deriviation indices. Usually before a scan you are on a fresh wallet with no
// addresses derived so we need to derive up to last active addresses the scan found
// before adding the transactions.
let (_, index_changeset) = graph.index.reveal_to_target_multi(&last_active_indices);
let mut indexed_tx_graph_changeset = graph.apply_update(graph_update);
indexed_tx_graph_changeset.append(index_changeset.into());
indexed_tx_graph_changeset
(chain.apply_update(update.local_chain)?, {
let (_, index_changeset) = graph
.index
.reveal_to_target_multi(&update.last_active_indices);
let mut indexed_tx_graph_changeset = graph.apply_update(update.tx_graph);
indexed_tx_graph_changeset.append(index_changeset.into());
indexed_tx_graph_changeset
})
}
EsploraCommands::Sync {
mut unused_spks,
@@ -231,12 +246,13 @@ fn main() -> anyhow::Result<()> {
let mut outpoints: Box<dyn Iterator<Item = OutPoint>> = Box::new(core::iter::empty());
let mut txids: Box<dyn Iterator<Item = Txid>> = Box::new(core::iter::empty());
let local_tip = chain.lock().expect("mutex must not be poisoned").tip();
// Get a short lock on the structures to get spks, utxos, and txs that we are interested
// in.
{
let graph = graph.lock().unwrap();
let chain = chain.lock().unwrap();
let chain_tip = chain.tip().block_id();
if *all_spks {
let all_spks = graph
@@ -276,7 +292,7 @@ fn main() -> anyhow::Result<()> {
let init_outpoints = graph.index.outpoints().iter().cloned();
let utxos = graph
.graph()
.filter_chain_unspents(&*chain, chain_tip, init_outpoints)
.filter_chain_unspents(&*chain, local_tip.block_id(), init_outpoints)
.map(|(_, utxo)| utxo)
.collect::<Vec<_>>();
outpoints = Box::new(
@@ -299,7 +315,7 @@ fn main() -> anyhow::Result<()> {
// `EsploraExt::update_tx_graph_without_keychain`.
let unconfirmed_txids = graph
.graph()
.list_chain_txs(&*chain, chain_tip)
.list_chain_txs(&*chain, local_tip.block_id())
.filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed())
.map(|canonical_tx| canonical_tx.tx_node.txid)
.collect::<Vec<Txid>>();
@@ -311,48 +327,30 @@ fn main() -> anyhow::Result<()> {
}
}
let mut graph_update =
client.sync(spks, txids, outpoints, scan_options.parallel_requests)?;
let mut update = client.sync(
local_tip,
spks,
txids,
outpoints,
scan_options.parallel_requests,
)?;
// Update last seen unconfirmed
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = graph_update.update_last_seen_unconfirmed(now);
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
graph.lock().unwrap().apply_update(graph_update)
(
chain.lock().unwrap().apply_update(update.local_chain)?,
graph.lock().unwrap().apply_update(update.tx_graph),
)
}
};
println!();
// Now that we're done updating the `IndexedTxGraph`, it's time to update the `LocalChain`! We
// want the `LocalChain` to have data about all the anchors in the `TxGraph` - for this reason,
// we want retrieve the blocks at the heights of the newly added anchors that are missing from
// our view of the chain.
let (missing_block_heights, tip) = {
let chain = &*chain.lock().unwrap();
let missing_block_heights = indexed_tx_graph_changeset
.graph
.missing_heights_from(chain)
.collect::<BTreeSet<_>>();
let tip = chain.tip();
(missing_block_heights, tip)
};
println!("prev tip: {}", tip.height());
println!("missing block heights: {:?}", missing_block_heights);
// Here, we actually fetch the missing blocks and create a `local_chain::Update`.
let chain_changeset = {
let chain_update = client
.update_local_chain(tip, missing_block_heights)
.context("scanning for blocks")?;
println!("new tip: {}", chain_update.tip.height());
chain.lock().unwrap().apply_update(chain_update)?
};
// We persist the changes
let mut db = db.lock().unwrap();
db.stage((chain_changeset, indexed_tx_graph_changeset));
db.stage((local_chain_changeset, indexed_tx_graph_changeset));
db.commit()?;
Ok(())
}

17
example-crates/wallet_esplora_async/src/main.rs
View File

@@ -53,18 +53,17 @@ async fn main() -> Result<(), anyhow::Error> {
(k, k_spks)
})
.collect();
let (mut update_graph, last_active_indices) = client
.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)
.await?;
let mut update = client
.full_scan(prev_tip, keychain_spks, STOP_GAP, PARALLEL_REQUESTS)
.await?;
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = update_graph.update_last_seen_unconfirmed(now);
let missing_heights = update_graph.missing_heights(wallet.local_chain());
let chain_update = client.update_local_chain(prev_tip, missing_heights).await?;
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
let update = Update {
last_active_indices,
graph: update_graph,
chain: Some(chain_update),
last_active_indices: update.last_active_indices,
graph: update.tx_graph,
chain: Some(update.local_chain),
};
wallet.apply_update(update)?;
wallet.commit()?;

25
example-crates/wallet_esplora_blocking/src/main.rs
View File

@@ -36,7 +36,6 @@ fn main() -> Result<(), anyhow::Error> {
let client =
esplora_client::Builder::new("https://blockstream.info/testnet/api").build_blocking();
let prev_tip = wallet.latest_checkpoint();
let keychain_spks = wallet
.all_unbounded_spk_iters()
.into_iter()
@@ -53,20 +52,20 @@ fn main() -> Result<(), anyhow::Error> {
})
.collect();
let (mut update_graph, last_active_indices) =
client.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)?;
let mut update = client.full_scan(
wallet.latest_checkpoint(),
keychain_spks,
STOP_GAP,
PARALLEL_REQUESTS,
)?;
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = update_graph.update_last_seen_unconfirmed(now);
let missing_heights = update_graph.missing_heights(wallet.local_chain());
let chain_update = client.update_local_chain(prev_tip, missing_heights)?;
let update = Update {
last_active_indices,
graph: update_graph,
chain: Some(chain_update),
};
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
wallet.apply_update(update)?;
wallet.apply_update(Update {
last_active_indices: update.last_active_indices,
graph: update.tx_graph,
chain: Some(update.local_chain),
})?;
wallet.commit()?;
println!();