machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/optimize-gbt-process-blocks

Browse Source
This commit is contained in:
softsimon
2024-09-22 12:41:51 +08:00
committed by GitHub
parent fbf27560b3 1038b4f908
commit 367ee68fe0
115 changed files with 2827 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/__tests__/api/common.ts
View File

@@ -1,5 +1,5 @@
import { Common } from '../../api/common';
import { MempoolTransactionExtended } from '../../mempool.interfaces';
import { MempoolTransactionExtended, TransactionExtended } from '../../mempool.interfaces';
const randomTransactions = require('./test-data/transactions-random.json');
const replacedTransactions = require('./test-data/transactions-replaced.json');
@@ -10,14 +10,14 @@ describe('Common', () => {
describe('RBF', () => {
const newTransactions = rbfTransactions.concat(randomTransactions);
test('should detect RBF transactions with fast method', () => {
const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions);
const result: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = Common.findRbfTransactions(newTransactions, replacedTransactions);
expect(Object.values(result).length).toEqual(2);
expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
});
test('should detect RBF transactions with scalable method', () => {
const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions, true);
const result: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = Common.findRbfTransactions(newTransactions, replacedTransactions, true);
expect(Object.values(result).length).toEqual(2);
expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');

22
backend/src/api/audit.ts
View File

@@ -2,6 +2,7 @@ import config from '../config';
import logger from '../logger';
import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
import transactionUtils from './transaction-utils';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
@@ -15,7 +16,8 @@ class Audit {
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const unseen: string[] = []; // present in the mined block, not in our mempool
const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
let prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
let deprioritized: string[] = []; // lower in the block than would be expected by in-band feerate alone
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
const accelerated: string[] = []; // prioritized by the mempool accelerator
@@ -133,23 +135,7 @@ class Audit {
totalWeight += tx.weight;
}
// identify "prioritized" transactions
let lastEffectiveRate = 0;
// Iterate over the mined template from bottom to top (excluding the coinbase)
// Transactions should appear in ascending order of mining priority.
for (let i = transactions.length - 1; i > 0; i--) {
const blockTx = transactions[i];
// If a tx has a lower in-band effective fee rate than the previous tx,
// it must have been prioritized out-of-band (in order to have a higher mining priority)
// so exclude from the analysis.
if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) {
prioritized.push(blockTx.txid);
// accelerated txs may or may not have their prioritized fee rate applied, so don't use them as a reference
} else if (!isAccelerated[blockTx.txid]) {
lastEffectiveRate = blockTx.effectiveFeePerVsize || 0;
}
}
({ prioritized, deprioritized } = transactionUtils.identifyPrioritizedTransactions(transactions, 'effectiveFeePerVsize'));
// transactions missing from near the end of our template are probably not being censored
let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);

1
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -323,6 +323,7 @@ class BitcoinApi implements AbstractBitcoinApi {
'witness_v1_taproot': 'v1_p2tr',
'nonstandard': 'nonstandard',
'multisig': 'multisig',
'anchor': 'anchor',
'nulldata': 'op_return'
};

144
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -20,6 +20,7 @@ import difficultyAdjustment from '../difficulty-adjustment';
import transactionRepository from '../../repositories/TransactionRepository';
import rbfCache from '../rbf-cache';
import { calculateMempoolTxCpfp } from '../cpfp';
import { handleError } from '../../utils/api';
class BitcoinRoutes {
public initRoutes(app: Application) {
@@ -86,7 +87,7 @@ class BitcoinRoutes {
res.set('Content-Type', 'application/json');
res.send(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -105,13 +106,13 @@ class BitcoinRoutes {
const result = mempoolBlocks.getMempoolBlocks();
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private getTransactionTimes(req: Request, res: Response) {
if (!Array.isArray(req.query.txId)) {
res.status(500).send('Not an array');
handleError(req, res, 500, 'Not an array');
return;
}
const txIds: string[] = [];
@@ -128,12 +129,12 @@ class BitcoinRoutes {
private async $getBatchedOutspends(req: Request, res: Response): Promise<IEsploraApi.Outspend[][] | void> {
const txids_csv = req.query.txids;
if (!txids_csv || typeof txids_csv !== 'string') {
res.status(500).send('Invalid txids format');
handleError(req, res, 500, 'Invalid txids format');
return;
}
const txids = txids_csv.split(',');
if (txids.length > 50) {
res.status(400).send('Too many txids requested');
handleError(req, res, 400, 'Too many txids requested');
return;
}
@@ -141,13 +142,13 @@ class BitcoinRoutes {
const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txids);
res.json(batchedOutspends);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async $getCpfpInfo(req: Request, res: Response) {
if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
res.status(501).send(`Invalid transaction ID.`);
handleError(req, res, 501, `Invalid transaction ID.`);
return;
}
@@ -180,7 +181,7 @@ class BitcoinRoutes {
try {
cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
} catch (e) {
res.status(500).send('failed to get CPFP info');
handleError(req, res, 500, 'failed to get CPFP info');
return;
}
}
@@ -209,7 +210,7 @@ class BitcoinRoutes {
if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
statusCode = 404;
}
res.status(statusCode).send(e instanceof Error ? e.message : e);
handleError(req, res, statusCode, e instanceof Error ? e.message : e);
}
}
@@ -223,7 +224,7 @@ class BitcoinRoutes {
if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
statusCode = 404;
}
res.status(statusCode).send(e instanceof Error ? e.message : e);
handleError(req, res, statusCode, e instanceof Error ? e.message : e);
}
}
@@ -284,13 +285,13 @@ class BitcoinRoutes {
// Not modified
// 422 Unprocessable Entity
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
res.status(422).send(`Psbt had no missing nonWitnessUtxos.`);
handleError(req, res, 422, `Psbt had no missing nonWitnessUtxos.`);
}
} catch (e: any) {
if (e instanceof Error && new RegExp(notFoundError).test(e.message)) {
res.status(404).send(e.message);
handleError(req, res, 404, e.message);
} else {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
}
@@ -304,7 +305,7 @@ class BitcoinRoutes {
if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
statusCode = 404;
}
res.status(statusCode).send(e instanceof Error ? e.message : e);
handleError(req, res, statusCode, e instanceof Error ? e.message : e);
}
}
@@ -314,7 +315,7 @@ class BitcoinRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
res.json(transactions);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -336,7 +337,7 @@ class BitcoinRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString());
res.json(block);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -346,7 +347,7 @@ class BitcoinRoutes {
res.setHeader('content-type', 'text/plain');
res.send(blockHeader);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -357,10 +358,11 @@ class BitcoinRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
res.json(auditSummary);
} else {
return res.status(404).send(`audit not available`);
handleError(req, res, 404, `audit not available`);
return;
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -371,7 +373,8 @@ class BitcoinRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
res.json(auditSummary);
} else {
return res.status(404).send(`transaction audit not available`);
handleError(req, res, 404, `transaction audit not available`);
return;
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
@@ -388,42 +391,49 @@ class BitcoinRoutes {
return await this.getLegacyBlocks(req, res);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getBlocksByBulk(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
return res.status(404).send(`This API is only available for Bitcoin networks`);
handleError(req, res, 404, `This API is only available for Bitcoin networks`);
return;
}
if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
handleError(req, res, 404, `This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
return;
}
if (!Common.indexingEnabled()) {
return res.status(404).send(`Indexing is required for this API`);
handleError(req, res, 404, `Indexing is required for this API`);
return;
}
const from = parseInt(req.params.from, 10);
if (!req.params.from || from < 0) {
return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
handleError(req, res, 400, `Parameter 'from' must be a block height (integer)`);
return;
}
const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
if (to < 0) {
return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
handleError(req, res, 400, `Parameter 'to' must be a block height (integer)`);
return;
}
if (from > to) {
return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
handleError(req, res, 400, `Parameter 'to' must be a higher block height than 'from'`);
return;
}
if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
handleError(req, res, 400, `You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
return;
}
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await blocks.$getBlocksBetweenHeight(from, to));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -458,10 +468,10 @@ class BitcoinRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(returnBlocks);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getBlockTransactions(req: Request, res: Response) {
try {
loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
@@ -483,7 +493,7 @@ class BitcoinRoutes {
res.json(transactions);
} catch (e) {
loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100);
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -492,13 +502,13 @@ class BitcoinRoutes {
const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10));
res.send(blockHash);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getAddress(req: Request, res: Response) {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
return;
}
@@ -507,15 +517,16 @@ class BitcoinRoutes {
res.json(addressData);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
return res.status(413).send(e instanceof Error ? e.message : e);
handleError(req, res, 413, e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getAddressTransactions(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
return;
}
@@ -528,23 +539,23 @@ class BitcoinRoutes {
res.json(transactions);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
res.status(413).send(e instanceof Error ? e.message : e);
handleError(req, res, 413, e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getAddressTransactionSummary(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND !== 'esplora') {
res.status(405).send('Address summary lookups require mempool/electrs backend.');
handleError(req, res, 405, 'Address summary lookups require mempool/electrs backend.');
return;
}
}
private async getScriptHash(req: Request, res: Response) {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
return;
}
@@ -555,15 +566,16 @@ class BitcoinRoutes {
res.json(addressData);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
return res.status(413).send(e instanceof Error ? e.message : e);
handleError(req, res, 413, e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getScriptHashTransactions(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
return;
}
@@ -578,16 +590,16 @@ class BitcoinRoutes {
res.json(transactions);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
res.status(413).send(e instanceof Error ? e.message : e);
handleError(req, res, 413, e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getScriptHashTransactionSummary(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND !== 'esplora') {
res.status(405).send('Scripthash summary lookups require mempool/electrs backend.');
handleError(req, res, 405, 'Scripthash summary lookups require mempool/electrs backend.');
return;
}
}
@@ -597,7 +609,7 @@ class BitcoinRoutes {
const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix);
res.send(blockHash);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -624,7 +636,7 @@ class BitcoinRoutes {
const rawMempool = await bitcoinApi.$getRawMempool();
res.send(rawMempool);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -632,12 +644,13 @@ class BitcoinRoutes {
try {
const result = blocks.getCurrentBlockHeight();
if (!result) {
return res.status(503).send(`Service Temporarily Unavailable`);
handleError(req, res, 503, `Service Temporarily Unavailable`);
return;
}
res.setHeader('content-type', 'text/plain');
res.send(result.toString());
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -647,7 +660,7 @@ class BitcoinRoutes {
res.setHeader('content-type', 'text/plain');
res.send(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -657,7 +670,7 @@ class BitcoinRoutes {
res.setHeader('content-type', 'application/octet-stream');
res.send(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -666,7 +679,7 @@ class BitcoinRoutes {
const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -675,7 +688,7 @@ class BitcoinRoutes {
const result = await bitcoinClient.validateAddress(req.params.address);
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -688,7 +701,7 @@ class BitcoinRoutes {
replaces
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -697,7 +710,7 @@ class BitcoinRoutes {
const result = rbfCache.getRbfTrees(false);
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -706,7 +719,7 @@ class BitcoinRoutes {
const result = rbfCache.getRbfTrees(true);
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -719,7 +732,7 @@ class BitcoinRoutes {
res.status(204).send();
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -728,7 +741,7 @@ class BitcoinRoutes {
const result = await bitcoinApi.$getOutspends(req.params.txId);
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -738,10 +751,10 @@ class BitcoinRoutes {
if (da) {
res.json(da);
} else {
res.status(503).send(`Service Temporarily Unavailable`);
handleError(req, res, 503, `Service Temporarily Unavailable`);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -752,7 +765,7 @@ class BitcoinRoutes {
const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx);
res.send(txIdResult);
} catch (e: any) {
res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
handleError(req, res, 400, e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
: (e.message || 'Error'));
}
}
@@ -764,7 +777,7 @@ class BitcoinRoutes {
const txIdResult = await bitcoinClient.sendRawTransaction(txHex);
res.send(txIdResult);
} catch (e: any) {
res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
handleError(req, res, 400, e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
: (e.message || 'Error'));
}
}
@@ -776,8 +789,7 @@ class BitcoinRoutes {
const result = await bitcoinApi.$testMempoolAccept(rawTxs, maxfeerate);
res.send(result);
} catch (e: any) {
res.setHeader('content-type', 'text/plain');
res.status(400).send(e.message && e.code ? 'testmempoolaccept RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
handleError(req, res, 400, e.message && e.code ? 'testmempoolaccept RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
: (e.message || 'Error'));
}
}

16
backend/src/api/blocks.ts
View File

@@ -219,10 +219,10 @@ class Blocks {
};
}
public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary {
public summarizeBlockTransactions(hash: string, height: number, transactions: TransactionExtended[]): BlockSummary {
return {
id: hash,
transactions: Common.classifyTransactions(transactions),
transactions: Common.classifyTransactions(transactions, height),
};
}
@@ -616,7 +616,7 @@ class Blocks {
// add CPFP
const cpfpSummary = calculateGoodBlockCpfp(height, txs, []);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2);
if (unclassifiedBlocks[height].version < 2 && targetSummaryVersion === 2) {
const cpfpClusters = await CpfpRepository.$getClustersAt(height);
@@ -653,7 +653,7 @@ class Blocks {
}
const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
for (const tx of classifiedTxs) {
classifiedTxMap[tx.txid] = tx;
@@ -912,7 +912,7 @@ class Blocks {
}
const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
if (Common.indexingEnabled()) {
@@ -1169,7 +1169,7 @@ class Blocks {
transactions: cpfpSummary.transactions.map(tx => {
let flags: number = 0;
try {
flags = Common.getTransactionFlags(tx);
flags = Common.getTransactionFlags(tx, height);
} catch (e) {
logger.warn('Failed to classify transaction: ' + (e instanceof Error ? e.message : e));
}
@@ -1188,7 +1188,7 @@ class Blocks {
} else {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(hash, txs);
summary = this.summarizeBlockTransactions(hash, height || 0, txs);
summaryVersion = 1;
} else {
// Call Core RPC
@@ -1324,7 +1324,7 @@ class Blocks {
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
summary = this.summarizeBlockTransactions(cleanBlock.hash, cleanBlock.height, txs);
summaryVersion = 1;
} else {
// Call Core RPC

83
backend/src/api/common.ts
View File

@@ -10,7 +10,6 @@ import logger from '../logger';
import { getVarIntLength, opcodes, parseMultisigScript } from '../utils/bitcoin-script';
// Bitcoin Core default policy settings
const TX_MAX_STANDARD_VERSION = 2;
const MAX_STANDARD_TX_WEIGHT = 400_000;
const MAX_BLOCK_SIGOPS_COST = 80_000;
const MAX_STANDARD_TX_SIGOPS_COST = (MAX_BLOCK_SIGOPS_COST / 5);
@@ -80,8 +79,8 @@ export class Common {
return arr;
}
static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: MempoolTransactionExtended[] } {
const matches: { [txid: string]: MempoolTransactionExtended[] } = {};
static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} {
const matches: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = {};
// For small N, a naive nested loop is extremely fast, but it doesn't scale
if (added.length < 1000 && deleted.length < 50 && !forceScalable) {
@@ -96,7 +95,7 @@ export class Common {
addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
});
if (foundMatches?.length) {
matches[addedTx.txid] = [...new Set(foundMatches)];
matches[addedTx.txid] = { replaced: [...new Set(foundMatches)], replacedBy: addedTx };
}
});
} else {
@@ -124,7 +123,7 @@ export class Common {
foundMatches.add(deletedTx);
}
if (foundMatches.size) {
matches[addedTx.txid] = [...foundMatches];
matches[addedTx.txid] = { replaced: [...foundMatches], replacedBy: addedTx };
}
}
}
@@ -139,17 +138,17 @@ export class Common {
const replaced: Set<MempoolTransactionExtended> = new Set();
for (let i = 0; i < tx.vin.length; i++) {
const vin = tx.vin[i];
const match = spendMap.get(`${vin.txid}:${vin.vout}`);
const key = `${vin.txid}:${vin.vout}`;
const match = spendMap.get(key);
if (match && match.txid !== tx.txid) {
replaced.add(match);
// remove this tx from the spendMap
// prevents the same tx being replaced more than once
for (const replacedVin of match.vin) {
const key = `${replacedVin.txid}:${replacedVin.vout}`;
spendMap.delete(key);
const replacedKey = `${replacedVin.txid}:${replacedVin.vout}`;
spendMap.delete(replacedKey);
}
}
const key = `${vin.txid}:${vin.vout}`;
spendMap.delete(key);
}
if (replaced.size) {
@@ -200,10 +199,13 @@ export class Common {
*
* returns true early if any standardness rule is violated, otherwise false
* (except for non-mandatory-script-verify-flag and p2sh script evaluation rules which are *not* enforced)
*
* As standardness rules change, we'll need to apply the rules in force *at the time* to older blocks.
* For now, just pull out individual rules into versioned functions where necessary.
*/
static isNonStandard(tx: TransactionExtended): boolean {
static isNonStandard(tx: TransactionExtended, height?: number): boolean {
// version
if (tx.version > TX_MAX_STANDARD_VERSION) {
if (this.isNonStandardVersion(tx, height)) {
return true;
}
@@ -250,6 +252,8 @@ export class Common {
}
} else if (['unknown', 'provably_unspendable', 'empty'].includes(vin.prevout?.scriptpubkey_type || '')) {
return true;
} else if (this.isNonStandardAnchor(tx, height)) {
return true;
}
// TODO: bad-witness-nonstandard
}
@@ -335,6 +339,49 @@ export class Common {
return false;
}
// Individual versioned standardness rules
static V3_STANDARDNESS_ACTIVATION_HEIGHT = {
'testnet4': 42_000,
'testnet': 2_900_000,
'signet': 211_000,
'': 863_500,
};
static isNonStandardVersion(tx: TransactionExtended, height?: number): boolean {
let TX_MAX_STANDARD_VERSION = 3;
if (
height != null
&& this.V3_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height <= this.V3_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
) {
// V3 transactions were non-standard to spend before v28.x (scheduled for 2024/09/30 https://github.com/bitcoin/bitcoin/issues/29891)
TX_MAX_STANDARD_VERSION = 2;
}
if (tx.version > TX_MAX_STANDARD_VERSION) {
return true;
}
return false;
}
static ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT = {
'testnet4': 42_000,
'testnet': 2_900_000,
'signet': 211_000,
'': 863_500,
};
static isNonStandardAnchor(tx: TransactionExtended, height?: number): boolean {
if (
height != null
&& this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height <= this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
) {
// anchor outputs were non-standard to spend before v28.x (scheduled for 2024/09/30 https://github.com/bitcoin/bitcoin/issues/29891)
return true;
}
return false;
}
static getNonWitnessSize(tx: TransactionExtended): number {
let weight = tx.weight;
let hasWitness = false;
@@ -415,7 +462,7 @@ export class Common {
return flags;
}
static getTransactionFlags(tx: TransactionExtended): number {
static getTransactionFlags(tx: TransactionExtended, height?: number): number {
let flags = tx.flags ? BigInt(tx.flags) : 0n;
// Update variable flags (CPFP, RBF)
@@ -548,7 +595,7 @@ export class Common {
if (hasFakePubkey) {
flags |= TransactionFlags.fake_pubkey;
}
// fast but bad heuristic to detect possible coinjoins
// (at least 5 inputs and 5 outputs, less than half of which are unique amounts, with no address reuse)
const addressReuse = Object.keys(reusedOutputAddresses).reduce((acc, key) => Math.max(acc, (reusedInputAddresses[key] || 0) + (reusedOutputAddresses[key] || 0)), 0) > 1;
@@ -564,17 +611,17 @@ export class Common {
flags |= TransactionFlags.batch_payout;
}
if (this.isNonStandard(tx)) {
if (this.isNonStandard(tx, height)) {
flags |= TransactionFlags.nonstandard;
}
return Number(flags);
}
static classifyTransaction(tx: TransactionExtended): TransactionClassified {
static classifyTransaction(tx: TransactionExtended, height?: number): TransactionClassified {
let flags = 0;
try {
flags = Common.getTransactionFlags(tx);
flags = Common.getTransactionFlags(tx, height);
} catch (e) {
logger.warn('Failed to add classification flags to transaction: ' + (e instanceof Error ? e.message : e));
}
@@ -585,8 +632,8 @@ export class Common {
};
}
static classifyTransactions(txs: TransactionExtended[]): TransactionClassified[] {
return txs.map(Common.classifyTransaction);
static classifyTransactions(txs: TransactionExtended[], height?: number): TransactionClassified[] {
return txs.map(tx => Common.classifyTransaction(tx, height));
}
static stripTransaction(tx: TransactionExtended): TransactionStripped {

29
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 81;
private static currentVersion = 82;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -700,6 +700,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD unseen_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(81);
}
if (databaseSchemaVersion < 82 && isBitcoin === true && config.MEMPOOL.NETWORK === 'mainnet') {
await this.$fixBadV1AuditBlocks();
await this.updateToSchemaVersion(82);
}
}
/**
@@ -1314,6 +1319,28 @@ class DatabaseMigration {
logger.warn(`Failed to migrate cpfp transaction data`);
}
}
private async $fixBadV1AuditBlocks(): Promise<void> {
const badBlocks = [
'000000000000000000011ad49227fc8c9ba0ca96ad2ebce41a862f9a244478dc',
'000000000000000000010ac1f68b3080153f2826ffddc87ceffdd68ed97d6960',
'000000000000000000024cbdafeb2660ae8bd2947d166e7fe15d1689e86b2cf7',
'00000000000000000002e1dbfbf6ae057f331992a058b822644b368034f87286',
'0000000000000000000019973b2778f08ad6d21e083302ff0833d17066921ebb',
];
for (const hash of badBlocks) {
try {
await this.$executeQuery(`
UPDATE blocks_audits
SET prioritized_txs = '[]'
WHERE hash = '${hash}'
`, true);
} catch (e) {
continue;
}
}
}
}
export default new DatabaseMigration();

1
backend/src/api/disk-cache.ts
View File

@@ -257,6 +257,7 @@ class DiskCache {
trees: rbfData.rbf.trees,
expiring: rbfData.rbf.expiring.map(([txid, value]) => ({ key: txid, value })),
mempool: memPool.getMempool(),
spendMap: memPool.getSpendMap(),
});
}
} catch (e) {

19
backend/src/api/explorer/channels.routes.ts
View File

@@ -1,6 +1,7 @@
import config from '../../config';
import { Application, Request, Response } from 'express';
import channelsApi from './channels.api';
import { handleError } from '../../utils/api';
class ChannelsRoutes {
constructor() { }
@@ -22,7 +23,7 @@ class ChannelsRoutes {
const channels = await channelsApi.$searchChannelsById(req.params.search);
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -38,7 +39,7 @@ class ChannelsRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channel);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -53,11 +54,11 @@ class ChannelsRoutes {
const status: string = typeof req.query.status === 'string' ? req.query.status : '';
if (index < -1) {
res.status(400).send('Invalid index');
handleError(req, res, 400, 'Invalid index');
return;
}
if (['open', 'active', 'closed'].includes(status) === false) {
res.status(400).send('Invalid status');
handleError(req, res, 400, 'Invalid status');
return;
}
@@ -69,14 +70,14 @@ class ChannelsRoutes {
res.header('X-Total-Count', channelsCount.toString());
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
try {
if (!Array.isArray(req.query.txId)) {
res.status(400).send('Not an array');
handleError(req, res, 400, 'Not an array');
return;
}
const txIds: string[] = [];
@@ -107,7 +108,7 @@ class ChannelsRoutes {
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -119,7 +120,7 @@ class ChannelsRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -132,7 +133,7 @@ class ChannelsRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}

8
backend/src/api/explorer/general.routes.ts
View File

@@ -3,6 +3,8 @@ import { Application, Request, Response } from 'express';
import nodesApi from './nodes.api';
import channelsApi from './channels.api';
import statisticsApi from './statistics.api';
import { handleError } from '../../utils/api';
class GeneralLightningRoutes {
constructor() { }
@@ -27,7 +29,7 @@ class GeneralLightningRoutes {
channels: channels,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -41,7 +43,7 @@ class GeneralLightningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -50,7 +52,7 @@ class GeneralLightningRoutes {
const statistics = await statisticsApi.$getLatestStatistics();
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
}

39
backend/src/api/explorer/nodes.routes.ts
View File

@@ -3,6 +3,7 @@ import { Application, Request, Response } from 'express';
import nodesApi from './nodes.api';
import DB from '../../database';
import { INodesRanking } from '../../mempool.interfaces';
import { handleError } from '../../utils/api';
class NodesRoutes {
constructor() { }
@@ -31,7 +32,7 @@ class NodesRoutes {
const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.params.search);
res.json(nodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -181,13 +182,13 @@ class NodesRoutes {
}
} catch (e) {}
}
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(nodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -195,7 +196,7 @@ class NodesRoutes {
try {
const node = await nodesApi.$getNode(req.params.public_key);
if (!node) {
res.status(404).send('Node not found');
handleError(req, res, 404, 'Node not found');
return;
}
res.header('Pragma', 'public');
@@ -203,7 +204,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(node);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -215,7 +216,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -223,7 +224,7 @@ class NodesRoutes {
try {
const node = await nodesApi.$getFeeHistogram(req.params.public_key);
if (!node) {
res.status(404).send('Node not found');
handleError(req, res, 404, 'Node not found');
return;
}
res.header('Pragma', 'public');
@@ -231,7 +232,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(node);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -247,7 +248,7 @@ class NodesRoutes {
topByChannels: topChannelsNodes,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -259,7 +260,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(topCapacityNodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -271,7 +272,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(topCapacityNodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -283,7 +284,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(topCapacityNodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -295,7 +296,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(nodesPerAs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -307,7 +308,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(worldNodes);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -322,7 +323,7 @@ class NodesRoutes {
);
if (country.length === 0) {
res.status(404).send(`This country does not exist or does not host any lightning nodes on clearnet`);
handleError(req, res, 404, `This country does not exist or does not host any lightning nodes on clearnet`);
return;
}
@@ -335,7 +336,7 @@ class NodesRoutes {
nodes: nodes,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -349,7 +350,7 @@ class NodesRoutes {
);
if (isp.length === 0) {
res.status(404).send(`This ISP does not exist or does not host any lightning nodes on clearnet`);
handleError(req, res, 404, `This ISP does not exist or does not host any lightning nodes on clearnet`);
return;
}
@@ -362,7 +363,7 @@ class NodesRoutes {
nodes: nodes,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -374,7 +375,7 @@ class NodesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(nodesPerAs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
}

35
backend/src/api/liquid/liquid.routes.ts
View File

@@ -3,6 +3,7 @@ import { Application, Request, Response } from 'express';
import config from '../../config';
import elementsParser from './elements-parser';
import icons from './icons';
import { handleError } from '../../utils/api';
class LiquidRoutes {
public initRoutes(app: Application) {
@@ -42,7 +43,7 @@ class LiquidRoutes {
res.setHeader('content-length', result.length);
res.send(result);
} else {
res.status(404).send('Asset icon not found');
handleError(req, res, 404, 'Asset icon not found');
}
}
@@ -51,7 +52,7 @@ class LiquidRoutes {
if (result) {
res.json(result);
} else {
res.status(404).send('Asset icons not found');
handleError(req, res, 404, 'Asset icons not found');
}
}
@@ -82,7 +83,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
res.json(pegs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -94,7 +95,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
res.json(reserves);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -106,7 +107,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(currentSupply);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -118,7 +119,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(currentReserves);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -130,7 +131,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(auditStatus);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -142,7 +143,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(federationAddresses);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -154,7 +155,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(federationAddresses);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -166,7 +167,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(federationUtxos);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -178,7 +179,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(expiredUtxos);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -190,7 +191,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(federationUtxos);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -202,7 +203,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(emergencySpentUtxos);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -214,7 +215,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(emergencySpentUtxos);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -226,7 +227,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(recentPegs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -238,7 +239,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(pegsVolume);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -250,7 +251,7 @@ class LiquidRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
res.json(pegsCount);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}

31
backend/src/api/mempool.ts
View File

@@ -19,12 +19,13 @@ class Mempool {
private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {};
private mempoolCandidates: { [txid: string ]: boolean } = {};
private spendMap = new Map<string, MempoolTransactionExtended>();
private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined;
deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
private accelerations: { [txId: string]: Acceleration } = {};
private accelerationPositions: { [txid: string]: { poolId: number, pool: string, block: number, vsize: number }[] } = {};
@@ -74,12 +75,12 @@ class Mempool {
}
public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; },
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void {
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void): void {
this.mempoolChangedCallback = fn;
}
public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[],
candidates?: GbtCandidates) => Promise<void>): void {
this.$asyncMempoolChangedCallback = fn;
}
@@ -362,12 +363,15 @@ class Mempool {
const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta);
this.recentlyDeleted.unshift(deletedTransactions);
this.recentlyDeleted.length = Math.min(this.recentlyDeleted.length, 10); // truncate to the last 10 mempool updates
if (this.mempoolChangedCallback && (hasChange || newTransactions.length || deletedTransactions.length)) {
this.mempoolChangedCallback(this.mempoolCache, newTransactions, this.recentlyDeleted, accelerationDelta);
}
if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length || candidatesChanged)) {
if (this.$asyncMempoolChangedCallback && (hasChange || newTransactions.length || deletedTransactions.length || candidatesChanged)) {
this.updateTimerProgress(timer, 'running async mempool callback');
await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta, candidates);
await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, this.recentlyDeleted, accelerationDelta, candidates);
this.updateTimerProgress(timer, 'completed async mempool callback');
}
@@ -541,16 +545,7 @@ class Mempool {
}
}
public handleRbfTransactions(rbfTransactions: { [txid: string]: MempoolTransactionExtended[]; }): void {
for (const rbfTransaction in rbfTransactions) {
if (this.mempoolCache[rbfTransaction] && rbfTransactions[rbfTransaction]?.length) {
// Store replaced transactions
rbfCache.add(rbfTransactions[rbfTransaction], this.mempoolCache[rbfTransaction]);
}
}
}
public handleMinedRbfTransactions(rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}): void {
public handleRbfTransactions(rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}): void {
for (const rbfTransaction in rbfTransactions) {
if (rbfTransactions[rbfTransaction].replacedBy && rbfTransactions[rbfTransaction]?.replaced?.length) {
// Store replaced transactions

73
backend/src/api/mining/mining-routes.ts
View File

@@ -10,6 +10,7 @@ import mining from "./mining";
import PricesRepository from '../../repositories/PricesRepository';
import AccelerationRepository from '../../repositories/AccelerationRepository';
import accelerationApi from '../services/acceleration';
import { handleError } from '../../utils/api';
class MiningRoutes {
public initRoutes(app: Application) {
@@ -53,12 +54,12 @@ class MiningRoutes {
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
if (['testnet', 'signet', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Prices are not available on testnets.');
handleError(req, res, 400, 'Prices are not available on testnets.');
return;
}
const timestamp = parseInt(req.query.timestamp as string, 10) || 0;
const currency = req.query.currency as string;
let response;
if (timestamp && currency) {
response = await PricesRepository.$getNearestHistoricalPrice(timestamp, currency);
@@ -71,7 +72,7 @@ class MiningRoutes {
}
res.status(200).send(response);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -84,9 +85,9 @@ class MiningRoutes {
res.json(stats);
} catch (e) {
if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
res.status(404).send(e.message);
handleError(req, res, 404, e.message);
} else {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
}
@@ -103,9 +104,9 @@ class MiningRoutes {
res.json(poolBlocks);
} catch (e) {
if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
res.status(404).send(e.message);
handleError(req, res, 404, e.message);
} else {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
}
@@ -129,7 +130,7 @@ class MiningRoutes {
res.json(pools);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -143,7 +144,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(stats);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -157,7 +158,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json(hashrates);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -172,9 +173,9 @@ class MiningRoutes {
res.json(hashrates);
} catch (e) {
if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
res.status(404).send(e.message);
handleError(req, res, 404, e.message);
} else {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
}
@@ -203,7 +204,7 @@ class MiningRoutes {
currentDifficulty: currentDifficulty,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -217,7 +218,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blockFees);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -235,7 +236,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blockFees);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -249,7 +250,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blockRewards);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -263,7 +264,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blockFeeRates);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -281,7 +282,7 @@ class MiningRoutes {
weights: blockWeights
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -293,7 +294,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -317,7 +318,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blocksHealth.map(health => [health.time, health.height, health.match_rate]));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -326,7 +327,7 @@ class MiningRoutes {
const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
if (!audit) {
res.status(204).send(`This block has not been audited.`);
handleError(req, res, 204, `This block has not been audited.`);
return;
}
@@ -335,7 +336,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
res.json(audit);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -358,7 +359,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -371,7 +372,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await BlocksAuditsRepository.$getBlockAuditScores(height, height - 15));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -384,7 +385,7 @@ class MiningRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
res.json(audit || 'null');
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -394,12 +395,12 @@ class MiningRoutes {
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
res.status(200).send(await AccelerationRepository.$getAccelerationInfo(req.params.slug));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -409,13 +410,13 @@ class MiningRoutes {
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, height));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -425,12 +426,12 @@ class MiningRoutes {
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, null, req.params.interval));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -440,12 +441,12 @@ class MiningRoutes {
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
res.status(200).send(await AccelerationRepository.$getAccelerationTotals(<string>req.query.pool, <string>req.query.interval));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -455,12 +456,12 @@ class MiningRoutes {
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
res.status(200).send(accelerationApi.accelerations || []);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
@@ -472,7 +473,7 @@ class MiningRoutes {
accelerationApi.accelerationRequested(req.params.txid);
res.status(200).send();
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
}

148
backend/src/api/rbf-cache.ts
View File

@@ -44,6 +44,22 @@ interface CacheEvent {
value?: any,
}
/**
* Singleton for tracking RBF trees
*
* Maintains a set of RBF trees, where each tree represents a sequence of
* consecutive RBF replacements.
*
* Trees are identified by the txid of the root transaction.
*
* To maintain consistency, the following invariants must be upheld:
* - Symmetry: replacedBy(A) = B <=> A in replaces(B)
* - Unique id: treeMap(treeMap(X)) = treeMap(X)
* - Unique tree: A in replaces(B) => treeMap(A) == treeMap(B)
* - Existence: X in treeMap => treeMap(X) in rbfTrees
* - Completeness: X in replacedBy => X in treeMap, Y in replaces => Y in treeMap
*/
class RbfCache {
private replacedBy: Map<string, string> = new Map();
private replaces: Map<string, string[]> = new Map();
@@ -61,6 +77,10 @@ class RbfCache {
setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
}
/**
* Low level cache operations
*/
private addTx(txid: string, tx: MempoolTransactionExtended): void {
this.txs.set(txid, tx);
this.cacheQueue.push({ op: CacheOp.Add, type: 'tx', txid });
@@ -92,6 +112,12 @@ class RbfCache {
this.cacheQueue.push({ op: CacheOp.Remove, type: 'exp', txid });
}
/**
* Basic data structure operations
* must uphold tree invariants
*/
public add(replaced: MempoolTransactionExtended[], newTxExtended: MempoolTransactionExtended): void {
if (!newTxExtended || !replaced?.length || this.txs.has(newTxExtended.txid)) {
return;
@@ -114,6 +140,10 @@ class RbfCache {
if (!replacedTx.rbf) {
txFullRbf = true;
}
if (this.replacedBy.has(replacedTx.txid)) {
// should never happen
continue;
}
this.replacedBy.set(replacedTx.txid, newTx.txid);
if (this.treeMap.has(replacedTx.txid)) {
const treeId = this.treeMap.get(replacedTx.txid);
@@ -140,18 +170,47 @@ class RbfCache {
}
}
newTx.fullRbf = txFullRbf;
const treeId = replacedTrees[0].tx.txid;
const newTree = {
tx: newTx,
time: newTime,
fullRbf: treeFullRbf,
replaces: replacedTrees
};
this.addTree(treeId, newTree);
this.updateTreeMap(treeId, newTree);
this.addTree(newTree.tx.txid, newTree);
this.updateTreeMap(newTree.tx.txid, newTree);
this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid));
}
public mined(txid): void {
if (!this.txs.has(txid)) {
return;
}
const treeId = this.treeMap.get(txid);
if (treeId && this.rbfTrees.has(treeId)) {
const tree = this.rbfTrees.get(treeId);
if (tree) {
this.setTreeMined(tree, txid);
tree.mined = true;
this.dirtyTrees.add(treeId);
this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId });
}
}
this.evict(txid);
}
// flag a transaction as removed from the mempool
public evict(txid: string, fast: boolean = false): void {
this.evictionCount++;
if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
this.addExpiration(txid, expiryTime);
}
}
/**
* Read-only public interface
*/
public has(txId: string): boolean {
return this.txs.has(txId);
}
@@ -232,32 +291,6 @@ class RbfCache {
return changes;
}
public mined(txid): void {
if (!this.txs.has(txid)) {
return;
}
const treeId = this.treeMap.get(txid);
if (treeId && this.rbfTrees.has(treeId)) {
const tree = this.rbfTrees.get(treeId);
if (tree) {
this.setTreeMined(tree, txid);
tree.mined = true;
this.dirtyTrees.add(treeId);
this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId });
}
}
this.evict(txid);
}
// flag a transaction as removed from the mempool
public evict(txid: string, fast: boolean = false): void {
this.evictionCount++;
if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
this.addExpiration(txid, expiryTime);
}
}
// is the transaction involved in a full rbf replacement?
public isFullRbf(txid: string): boolean {
const treeId = this.treeMap.get(txid);
@@ -271,6 +304,10 @@ class RbfCache {
return tree?.fullRbf;
}
/**
* Cache maintenance & utility functions
*/
private cleanup(): void {
const now = Date.now();
for (const txid of this.expiring.keys()) {
@@ -299,10 +336,6 @@ class RbfCache {
for (const tx of (replaces || [])) {
// recursively remove prior versions from the cache
this.replacedBy.delete(tx);
// if this is the id of a tree, remove that too
if (this.treeMap.get(tx) === tx) {
this.removeTree(tx);
}
this.remove(tx);
}
}
@@ -370,14 +403,21 @@ class RbfCache {
};
}
public async load({ txs, trees, expiring, mempool }): Promise<void> {
public async load({ txs, trees, expiring, mempool, spendMap }): Promise<void> {
try {
txs.forEach(txEntry => {
this.txs.set(txEntry.value.txid, txEntry.value);
});
this.staleCount = 0;
for (const deflatedTree of trees) {
await this.importTree(mempool, deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
for (const deflatedTree of trees.sort((a, b) => Object.keys(b).length - Object.keys(a).length)) {
const tree = await this.importTree(mempool, deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
if (tree) {
this.addTree(tree.tx.txid, tree);
this.updateTreeMap(tree.tx.txid, tree);
if (tree.mined) {
this.evict(tree.tx.txid);
}
}
}
expiring.forEach(expiringEntry => {
if (this.txs.has(expiringEntry.key)) {
@@ -385,6 +425,31 @@ class RbfCache {
}
});
this.staleCount = 0;
// connect cached trees to current mempool transactions
const conflicts: Record<string, { replacedBy: MempoolTransactionExtended, replaces: Set<MempoolTransactionExtended> }> = {};
for (const tree of this.rbfTrees.values()) {
const tx = this.getTx(tree.tx.txid);
if (!tx || tree.mined) {
continue;
}
for (const vin of tx.vin) {
const conflict = spendMap.get(`${vin.txid}:${vin.vout}`);
if (conflict && conflict.txid !== tx.txid) {
if (!conflicts[conflict.txid]) {
conflicts[conflict.txid] = {
replacedBy: conflict,
replaces: new Set(),
};
}
conflicts[conflict.txid].replaces.add(tx);
}
}
}
for (const { replacedBy, replaces } of Object.values(conflicts)) {
this.add([...replaces.values()], replacedBy);
}
await this.checkTrees();
logger.debug(`loaded ${txs.length} txs, ${trees.length} trees into rbf cache, ${expiring.length} due to expire, ${this.staleCount} were stale`);
this.cleanup();
@@ -426,6 +491,12 @@ class RbfCache {
return;
}
// if this tx is already in the cache, return early
if (this.treeMap.has(txid)) {
this.removeTree(deflated.key);
return;
}
// recursively reconstruct child trees
for (const childId of treeInfo.replaces) {
const replaced = await this.importTree(mempool, root, childId, deflated, txs, mined);
@@ -457,10 +528,6 @@ class RbfCache {
fullRbf: treeInfo.fullRbf,
replaces,
};
this.treeMap.set(txid, root);
if (root === txid) {
this.addTree(root, tree);
}
return tree;
}
@@ -511,6 +578,7 @@ class RbfCache {
processTxs(txs);
}
// evict missing transactions
for (const txid of txids) {
if (!found[txid]) {
this.evict(txid, false);

1
backend/src/api/redis-cache.ts
View File

@@ -365,6 +365,7 @@ class RedisCache {
trees: rbfTrees.map(loadedTree => { loadedTree.value.key = loadedTree.key; return loadedTree.value; }),
expiring: rbfExpirations,
mempool: memPool.getMempool(),
spendMap: memPool.getSpendMap(),
});
}

81
backend/src/api/transaction-utils.ts
View File

@@ -338,6 +338,87 @@ class TransactionUtils {
const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
return witness[positionOfScript];
}
// calculate the most parsimonious set of prioritizations given a list of block transactions
// (i.e. the most likely prioritizations and deprioritizations)
public identifyPrioritizedTransactions(transactions: any[], rateKey: string): { prioritized: string[], deprioritized: string[] } {
// find the longest increasing subsequence of transactions
// (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms)
// should be O(n log n)
const X = transactions.slice(1).reverse().map((tx) => ({ txid: tx.txid, rate: tx[rateKey] })); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase)
if (X.length < 2) {
return { prioritized: [], deprioritized: [] };
}
const N = X.length;
const P: number[] = new Array(N);
const M: number[] = new Array(N + 1);
M[0] = -1; // undefined so can be set to any value
let L = 0;
for (let i = 0; i < N; i++) {
// Binary search for the smallest positive l ≤ L
// such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize
let lo = 1;
let hi = L + 1;
while (lo < hi) {
const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi
if (X[M[mid]].rate > X[i].rate) {
hi = mid;
} else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize
lo = mid + 1;
}
}
// After searching, lo == hi is 1 greater than the
// length of the longest prefix of X[i]
const newL = lo;
// The predecessor of X[i] is the last index of
// the subsequence of length newL-1
P[i] = M[newL - 1];
M[newL] = i;
if (newL > L) {
// If we found a subsequence longer than any we've
// found yet, update L
L = newL;
}
}
// Reconstruct the longest increasing subsequence
// It consists of the values of X at the L indices:
// ..., P[P[M[L]]], P[M[L]], M[L]
const LIS: any[] = new Array(L);
let k = M[L];
for (let j = L - 1; j >= 0; j--) {
LIS[j] = X[k];
k = P[k];
}
const lisMap = new Map<string, number>();
LIS.forEach((tx, index) => lisMap.set(tx.txid, index));
const prioritized: string[] = [];
const deprioritized: string[] = [];
let lastRate = X[0].rate;
for (const tx of X) {
if (lisMap.has(tx.txid)) {
lastRate = tx.rate;
} else {
if (Math.abs(tx.rate - lastRate) < 0.1) {
// skip if the rate is almost the same as the previous transaction
} else if (tx.rate <= lastRate) {
prioritized.push(tx.txid);
} else {
deprioritized.push(tx.txid);
}
}
}
return { prioritized, deprioritized };
}
}
export default new TransactionUtils();

19
backend/src/api/websocket-handler.ts
View File

@@ -520,8 +520,17 @@ class WebsocketHandler {
}
}
/**
*
* @param newMempool
* @param mempoolSize
* @param newTransactions array of transactions added this mempool update.
* @param recentlyDeletedTransactions array of arrays of transactions removed in the last N mempool updates, most recent first.
* @param accelerationDelta
* @param candidates
*/
async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
newTransactions: MempoolTransactionExtended[], recentlyDeletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[],
candidates?: GbtCandidates): Promise<void> {
if (!this.webSocketServers.length) {
throw new Error('No WebSocket.Server have been set');
@@ -529,6 +538,8 @@ class WebsocketHandler {
this.printLogs();
const deletedTransactions = recentlyDeletedTransactions.length ? recentlyDeletedTransactions[0] : [];
const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool);
let added = newTransactions;
let removed = deletedTransactions;
@@ -547,7 +558,7 @@ class WebsocketHandler {
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
const mempoolInfo = memPool.getMempoolInfo();
const vBytesPerSecond = memPool.getVBytesPerSecond();
const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
const rbfTransactions = Common.findRbfTransactions(newTransactions, recentlyDeletedTransactions.flat());
const da = difficultyAdjustment.getDifficultyAdjustment();
const accelerations = memPool.getAccelerations();
memPool.handleRbfTransactions(rbfTransactions);
@@ -578,7 +589,7 @@ class WebsocketHandler {
const replacedTransactions: { replaced: string, by: TransactionExtended }[] = [];
for (const tx of newTransactions) {
if (rbfTransactions[tx.txid]) {
for (const replaced of rbfTransactions[tx.txid]) {
for (const replaced of rbfTransactions[tx.txid].replaced) {
replacedTransactions.push({ replaced: replaced.txid, by: tx });
}
}
@@ -947,7 +958,7 @@ class WebsocketHandler {
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.handleRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
if (config.MEMPOOL.AUDIT && memPool.isInSync()) {

5
backend/src/repositories/BlocksAuditsRepository.ts
View File

@@ -132,11 +132,12 @@ class BlocksAuditRepositories {
firstSeen = tx.time;
}
});
const wasSeen = blockAudit.version === 1 ? !blockAudit.unseenTxs.includes(txid) : (isExpected || isPrioritized || isAccelerated);
return {
seen: isExpected || isPrioritized || isAccelerated,
seen: wasSeen,
expected: isExpected,
added: isAdded,
added: isAdded && (blockAudit.version === 0 || !wasSeen),
prioritized: isPrioritized,
conflict: isConflict,
accelerated: isAccelerated,

2
backend/src/repositories/BlocksRepository.ts
View File

@@ -1106,7 +1106,7 @@ class BlocksRepository {
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
summary = blocks.summarizeBlockTransactions(dbBlk.id, txs);
summary = blocks.summarizeBlockTransactions(dbBlk.id, dbBlk.height, txs);
summaryVersion = 1;
} else {
// Call Core RPC

9
backend/src/utils/api.ts Normal file
View File

@@ -0,0 +1,9 @@
import { Request, Response } from 'express';
export function handleError(req: Request, res: Response, statusCode: number, errorMessage: string | unknown): void {
if (req.accepts('json')) {
res.status(statusCode).json({ error: errorMessage });
} else {
res.status(statusCode).send(errorMessage);
}
}

4
backend/src/utils/bitcoin-script.ts
View File

@@ -158,7 +158,7 @@ export function parseMultisigScript(script: string): void | { m: number, n: numb
if (!opN) {
return;
}
if (!opN.startsWith('OP_PUSHNUM_')) {
if (opN !== 'OP_0' && !opN.startsWith('OP_PUSHNUM_')) {
return;
}
const n = parseInt(opN.match(/[0-9]+/)?.[0] || '', 10);
@@ -178,7 +178,7 @@ export function parseMultisigScript(script: string): void | { m: number, n: numb
if (!opM) {
return;
}
if (!opM.startsWith('OP_PUSHNUM_')) {
if (opM !== 'OP_0' && !opM.startsWith('OP_PUSHNUM_')) {
return;
}
const m = parseInt(opM.match(/[0-9]+/)?.[0] || '', 10);