machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement v1 block audits

Browse Source
This commit is contained in:
Mononaut
2024-07-20 11:55:11 +00:00
parent 96e2e6060b
commit 7cc01af631
10 changed files with 134 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
backend/src/api/audit.ts
View File

@@ -6,15 +6,16 @@ import rbfCache from './rbf-cache';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
class Audit {
auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false)
: { censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
: { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
}
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const prioritized: string[] = [] // present in the mined block, not in the template, but further down in the mempool
const unseen: string[] = []; // present in the mined block, not in our mempool
const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
const accelerated: string[] = []; // prioritized by the mempool accelerator
@@ -113,11 +114,16 @@ class Audit {
} else {
if (rbfCache.has(tx.txid)) {
rbf.push(tx.txid);
} else if (!isDisplaced[tx.txid]) {
if (!mempool[tx.txid] && !rbfCache.getReplacedBy(tx.txid)) {
unseen.push(tx.txid);
}
} else {
if (mempool[tx.txid]) {
prioritized.push(tx.txid);
if (isDisplaced[tx.txid]) {
added.push(tx.txid);
}
} else {
added.push(tx.txid);
unseen.push(tx.txid);
}
}
overflowWeight += tx.weight;
@@ -125,6 +131,23 @@ class Audit {
totalWeight += tx.weight;
}
// identify "prioritized" transactions
let lastEffectiveRate = 0;
// Iterate over the mined template from bottom to top (excluding the coinbase)
// Transactions should appear in ascending order of mining priority.
for (let i = transactions.length - 1; i > 0; i--) {
const blockTx = transactions[i];
// If a tx has a lower in-band effective fee rate than the previous tx,
// it must have been prioritized out-of-band (in order to have a higher mining priority)
// so exclude from the analysis.
if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) {
prioritized.push(blockTx.txid);
} else {
lastEffectiveRate = blockTx.effectiveFeePerVsize || 0;
}
}
// transactions missing from near the end of our template are probably not being censored
let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
let maxOverflowRate = 0;
@@ -165,6 +188,7 @@ class Audit {
const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
return {
unseen,
censored: Object.keys(isCensored),
added,
prioritized,

18
backend/src/api/blocks.ts
View File

@@ -439,7 +439,7 @@ class Blocks {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
if (cpfpSummary) {
await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
@@ -927,12 +927,12 @@ class Blocks {
const newBlock = await this.$indexBlock(lastBlock.height - i);
this.blocks.push(newBlock);
this.updateTimerProgress(timer, `reindexed block`);
let cpfpSummary;
let newCpfpSummary;
if (config.MEMPOOL.CPFP_INDEXING) {
cpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
newCpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
this.updateTimerProgress(timer, `reindexed block cpfp`);
}
await this.$getStrippedBlockTransactions(newBlock.id, true, true, cpfpSummary, newBlock.height);
await this.$getStrippedBlockTransactions(newBlock.id, true, true, newCpfpSummary, newBlock.height);
this.updateTimerProgress(timer, `reindexed block summary`);
}
await mining.$indexDifficultyAdjustments();
@@ -981,7 +981,7 @@ class Blocks {
// start async callbacks
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions));
if (block.height % 2016 === 0) {
if (Common.indexingEnabled()) {
@@ -1178,7 +1178,7 @@ class Blocks {
};
}),
};
summaryVersion = 1;
summaryVersion = cpfpSummary.version;
} else {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
@@ -1397,11 +1397,11 @@ class Blocks {
return this.currentBlockHeight;
}
public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary | null> {
public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[]): Promise<CpfpSummary | null> {
let transactions = txs;
if (!transactions) {
if (config.MEMPOOL.BACKEND === 'esplora') {
transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
}
if (!transactions) {
const block = await bitcoinClient.getBlock(hash, 2);
@@ -1413,7 +1413,7 @@ class Blocks {
}
if (transactions?.length != null) {
const summary = calculateFastBlockCpfp(height, transactions as TransactionExtended[]);
const summary = calculateFastBlockCpfp(height, transactions);
await this.$saveCpfp(hash, height, summary);

2
backend/src/api/common.ts
View File

@@ -1,6 +1,6 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { Request } from 'express';
import { CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
import { isIP } from 'net';

4
backend/src/api/cpfp.ts
View File

@@ -6,7 +6,7 @@ import { Acceleration } from './acceleration/acceleration';
const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
const MAX_CLUSTER_ITERATIONS = 100;
export function calculateFastBlockCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
@@ -93,6 +93,7 @@ export function calculateFastBlockCpfp(height: number, transactions: Transaction
return {
transactions,
clusters,
version: 1,
};
}
@@ -159,6 +160,7 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
return {
transactions: transactions.map(tx => txMap[tx.txid]),
clusters: clusterArray,
version: 2,
};
}

2
backend/src/api/mini-miner.ts
View File

@@ -337,7 +337,7 @@ export function makeBlockTemplate(candidates: MempoolTransactionExtended[], acce
let failures = 0;
while (mempoolArray.length || modified.length) {
// skip invalid transactions
while (mempoolArray[0].used || mempoolArray[0].modified) {
while (mempoolArray[0]?.used || mempoolArray[0]?.modified) {
mempoolArray.shift();
}

10
backend/src/api/websocket-handler.ts
View File

@@ -3,7 +3,7 @@ import * as WebSocket from 'ws';
import {
BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
MempoolBlockDelta, MempoolDelta, MempoolDeltaTxids
MempoolDelta, MempoolDeltaTxids
} from '../mempool.interfaces';
import blocks from './blocks';
import memPool from './mempool';
@@ -933,6 +933,8 @@ class WebsocketHandler {
throw new Error('No WebSocket.Server have been set');
}
const blockTransactions = structuredClone(transactions);
this.printLogs();
await statistics.runStatistics();
@@ -942,7 +944,7 @@ class WebsocketHandler {
let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
const accelerations = Object.values(mempool.getAccelerations());
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
@@ -962,7 +964,7 @@ class WebsocketHandler {
}
if (Common.indexingEnabled()) {
const { censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -984,9 +986,11 @@ class WebsocketHandler {
});
BlocksAuditsRepository.$saveAudit({
version: 1,
time: block.timestamp,
height: block.height,
hash: block.id,
unseenTxs: unseen,
addedTxs: added,
prioritizedTxs: prioritized,
missingTxs: censored,