machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/optimize-mempool-block-7

Browse Source
This commit is contained in:
softsimon
2023-03-04 10:34:46 +09:00
committed by GitHub
parent b56f83a134 637045acde
commit 1bbdcdafa1
163 changed files with 10262 additions and 4585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/api/audit.ts
View File

@@ -119,7 +119,8 @@ class Audit {
}
const numCensored = Object.keys(isCensored).length;
const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
const numMatches = matches.length - 1; // adjust for coinbase tx
const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
return {
censored: Object.keys(isCensored),

31
backend/src/api/bitcoin/bitcoin-api.interface.ts
View File

@@ -172,4 +172,35 @@ export namespace IBitcoinApi {
}
}
export interface BlockStats {
"avgfee": number;
"avgfeerate": number;
"avgtxsize": number;
"blockhash": string;
"feerate_percentiles": [number, number, number, number, number];
"height": number;
"ins": number;
"maxfee": number;
"maxfeerate": number;
"maxtxsize": number;
"medianfee": number;
"mediantime": number;
"mediantxsize": number;
"minfee": number;
"minfeerate": number;
"mintxsize": number;
"outs": number;
"subsidy": number;
"swtotal_size": number;
"swtotal_weight": number;
"swtxs": number;
"time": number;
"total_out": number;
"total_size": number;
"total_weight": number;
"totalfee": number;
"txs": number;
"utxo_increase": number;
"utxo_size_inc": number;
}
}

1
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
mediantime: block.mediantime,
};
}

47
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -95,6 +95,8 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
.post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
;
if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -215,7 +217,15 @@ class BitcoinRoutes {
res.json(cpfpInfo);
return;
} else {
const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
let cpfpInfo;
if (config.DATABASE.ENABLED) {
cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
} else {
res.json({
ancestors: []
});
return;
}
if (cpfpInfo) {
res.json(cpfpInfo);
return;
@@ -402,6 +412,41 @@ class BitcoinRoutes {
}
}
private async getBlocksByBulk(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
return res.status(404).send(`This API is only available for Bitcoin networks`);
}
if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
}
if (!Common.indexingEnabled()) {
return res.status(404).send(`Indexing is required for this API`);
}
const from = parseInt(req.params.from, 10);
if (!req.params.from || from < 0) {
return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
}
const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
if (to < 0) {
return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
}
if (from > to) {
return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
}
if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
}
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await blocks.$getBlocksBetweenHeight(from, to));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getLegacyBlocks(req: Request, res: Response) {
try {
const returnBlocks: IEsploraApi.Block[] = [];

1
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@@ -88,6 +88,7 @@ export namespace IEsploraApi {
size: number;
weight: number;
previousblockhash: string;
mediantime: number;
}
export interface Address {

304
backend/src/api/blocks.ts
View File

@@ -2,7 +2,7 @@ import config from '../config';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
@@ -13,7 +13,6 @@ import poolsRepository from '../repositories/PoolsRepository';
import blocksRepository from '../repositories/BlocksRepository';
import loadingIndicators from './loading-indicators';
import BitcoinApi from './bitcoin/bitcoin-api';
import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
@@ -25,6 +24,7 @@ import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
import chainTips from './chain-tips';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -142,7 +142,7 @@ class Blocks {
* @param block
* @returns BlockSummary
*/
private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
const stripped = block.tx.map((tx) => {
return {
txid: tx.txid,
@@ -165,33 +165,81 @@ class Blocks {
* @returns BlockExtended
*/
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
blockExtended.extras.usd = priceUpdater.latestPrices.USD;
const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const blk: Partial<BlockExtended> = Object.assign({}, block);
const extras: Partial<BlockExtension> = {};
extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
if (block.height === 0) {
blockExtended.extras.medianFee = 0; // 50th percentiles
blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
blockExtended.extras.totalFees = 0;
blockExtended.extras.avgFee = 0;
blockExtended.extras.avgFeeRate = 0;
extras.medianFee = 0; // 50th percentiles
extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
extras.totalFees = 0;
extras.avgFee = 0;
extras.avgFeeRate = 0;
extras.utxoSetChange = 0;
extras.avgTxSize = 0;
extras.totalInputs = 0;
extras.totalOutputs = 1;
extras.totalOutputAmt = 0;
extras.segwitTotalTxs = 0;
extras.segwitTotalSize = 0;
extras.segwitTotalWeight = 0;
} else {
const stats = await bitcoinClient.getBlockStats(block.id, [
'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
]);
blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
blockExtended.extras.totalFees = stats.totalfee;
blockExtended.extras.avgFee = stats.avgfee;
blockExtended.extras.avgFeeRate = stats.avgfeerate;
const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
extras.totalFees = stats.totalfee;
extras.avgFee = stats.avgfee;
extras.avgFeeRate = stats.avgfeerate;
extras.utxoSetChange = stats.utxo_increase;
extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
extras.totalInputs = stats.ins;
extras.totalOutputs = stats.outs;
extras.totalOutputAmt = stats.total_out;
extras.segwitTotalTxs = stats.swtxs;
extras.segwitTotalSize = stats.swtotal_size;
extras.segwitTotalWeight = stats.swtotal_weight;
}
if (Common.blocksSummariesIndexingEnabled()) {
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
if (extras.feePercentiles !== null) {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
extras.virtualSize = block.weight / 4.0;
if (coinbaseTx?.vout.length > 0) {
extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
} else {
extras.coinbaseAddress = null;
extras.coinbaseSignature = null;
extras.coinbaseSignatureAscii = null;
}
const header = await bitcoinClient.getBlockHeader(block.id, false);
extras.header = header;
const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
extras.utxoSetSize = txoutset.txouts,
extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
} else {
extras.utxoSetSize = null;
extras.totalInputAmt = null;
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
let pool: PoolTag;
if (blockExtended.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
if (coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(coinbaseTx);
} else {
if (config.DATABASE.ENABLED === true) {
pool = await poolsRepository.$getUnknownPool();
@@ -201,25 +249,27 @@ class Blocks {
}
if (!pool) { // We should never have this situation in practise
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
`Check your "pools" table entries`);
} else {
blockExtended.extras.pool = {
id: pool.id,
extras.pool = {
id: pool.uniqueId,
name: pool.name,
slug: pool.slug,
};
}
extras.matchRate = null;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blockExtended.extras.matchRate = auditScore.matchRate;
extras.matchRate = auditScore.matchRate;
}
}
}
return blockExtended;
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
}
/**
@@ -245,15 +295,18 @@ class Blocks {
} else {
pools = poolsParser.miningPools;
}
for (let i = 0; i < pools.length; ++i) {
if (address !== undefined) {
const addresses: string[] = JSON.parse(pools[i].addresses);
const addresses: string[] = typeof pools[i].addresses === 'string' ?
JSON.parse(pools[i].addresses) : pools[i].addresses;
if (addresses.indexOf(address) !== -1) {
return pools[i];
}
}
const regexes: string[] = JSON.parse(pools[i].regexes);
const regexes: string[] = typeof pools[i].regexes === 'string' ?
JSON.parse(pools[i].regexes) : pools[i].regexes;
for (let y = 0; y < regexes.length; ++y) {
const regex = new RegExp(regexes[y], 'i');
const match = asciiScriptSig.match(regex);
@@ -431,7 +484,7 @@ class Blocks {
loadingIndicators.setProgress('block-indexing', progress, false);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -479,13 +532,13 @@ class Blocks {
if (blockchainInfo.blocks === blockchainInfo.headers) {
const heightDiff = blockHeightTip % 2016;
const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty;
if (blockHeightTip >= 2016) {
const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
logger.debug(`Initial difficulty adjustment data set.`);
}
@@ -500,6 +553,7 @@ class Blocks {
} else {
this.currentBlockHeight++;
logger.debug(`New block found (#${this.currentBlockHeight})!`);
await chainTips.updateOrphanedBlocks();
}
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -516,18 +570,18 @@ class Blocks {
if (Common.indexingEnabled()) {
if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
// We assume there won't be a reorg with more than 10 block depth
await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries();
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock['height'] - i);
const newBlock = await this.$indexBlock(lastBlock.height - i);
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
if (config.MEMPOOL.CPFP_INDEXING) {
await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
await this.$indexCPFP(newBlock.id, lastBlock.height - i);
}
}
await mining.$indexDifficultyAdjustments();
@@ -603,12 +657,12 @@ class Blocks {
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock !== null) {
return prepareBlock(dbBlock);
return dbBlock;
}
}
const blockHash = await bitcoinApi.$getBlockHash(height);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -616,11 +670,11 @@ class Blocks {
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return prepareBlock(blockExtended);
return blockExtended;
}
/**
* Index a block by hash if it's missing from the database. Returns the block after indexing
* Get one block by its hash
*/
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
// Check the memory cache
@@ -629,31 +683,14 @@ class Blocks {
return blockByHash;
}
// Block has already been indexed
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHash(hash);
if (dbBlock != null) {
return prepareBlock(dbBlock);
}
}
// Not Bitcoin network, return the block as it
// Not Bitcoin network, return the block as it from the bitcoin backend
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return await bitcoinApi.$getBlock(hash);
}
let block = await bitcoinClient.getBlock(hash);
block = prepareBlock(block);
// Bitcoin network, add our custom data on top
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
if (Common.indexingEnabled()) {
delete(blockExtended['coinbaseTx']);
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return blockExtended;
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
return await this.$indexBlock(block.height);
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@@ -687,8 +724,19 @@ class Blocks {
return summary.transactions;
}
/**
* Get 15 blocks
*
* Internally this function uses two methods to get the blocks, and
* the method is automatically selected:
* - Using previous block hash links
* - Using block height
*
* @param fromHeight
* @param limit
* @returns
*/
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
if (currentHeight > this.currentBlockHeight) {
limit -= currentHeight - this.currentBlockHeight;
@@ -700,27 +748,15 @@ class Blocks {
return returnBlocks;
}
// Check if block height exist in local cache to skip the hash lookup
const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
let startFromHash: string | null = null;
if (blockByHeight) {
startFromHash = blockByHeight.id;
} else if (!Common.indexingEnabled()) {
startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
}
let nextHash = startFromHash;
for (let i = 0; i < limit && currentHeight >= 0; i++) {
let block = this.getBlocks().find((b) => b.height === currentHeight);
if (block) {
// Using the memory cache (find by height)
returnBlocks.push(block);
} else if (Common.indexingEnabled()) {
} else {
// Using indexing (find by height, index on the fly, save in database)
block = await this.$indexBlock(currentHeight);
returnBlocks.push(block);
} else if (nextHash != null) {
block = await this.$indexBlock(currentHeight);
nextHash = block.previousblockhash;
returnBlocks.push(block);
}
currentHeight--;
}
@@ -728,6 +764,114 @@ class Blocks {
return returnBlocks;
}
/**
* Used for bulk block data query
*
* @param fromHeight
* @param toHeight
*/
public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
if (!Common.indexingEnabled()) {
return [];
}
const blocks: any[] = [];
while (fromHeight <= toHeight) {
let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
await this.$indexBlock(fromHeight);
block = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
continue;
}
}
// Cleanup fields before sending the response
const cleanBlock: any = {
height: block.height ?? null,
hash: block.id ?? null,
timestamp: block.timestamp ?? null,
median_timestamp: block.mediantime ?? null,
previous_block_hash: block.previousblockhash ?? null,
difficulty: block.difficulty ?? null,
header: block.extras.header ?? null,
version: block.version ?? null,
bits: block.bits ?? null,
nonce: block.nonce ?? null,
size: block.size ?? null,
weight: block.weight ?? null,
tx_count: block.tx_count ?? null,
merkle_root: block.merkle_root ?? null,
reward: block.extras.reward ?? null,
total_fee_amt: block.extras.totalFees ?? null,
avg_fee_amt: block.extras.avgFee ?? null,
median_fee_amt: block.extras.medianFeeAmt ?? null,
fee_amt_percentiles: block.extras.feePercentiles ?? null,
avg_fee_rate: block.extras.avgFeeRate ?? null,
median_fee_rate: block.extras.medianFee ?? null,
fee_rate_percentiles: block.extras.feeRange ?? null,
total_inputs: block.extras.totalInputs ?? null,
total_input_amt: block.extras.totalInputAmt ?? null,
total_outputs: block.extras.totalOutputs ?? null,
total_output_amt: block.extras.totalOutputAmt ?? null,
segwit_total_txs: block.extras.segwitTotalTxs ?? null,
segwit_total_size: block.extras.segwitTotalSize ?? null,
segwit_total_weight: block.extras.segwitTotalWeight ?? null,
avg_tx_size: block.extras.avgTxSize ?? null,
utxoset_change: block.extras.utxoSetChange ?? null,
utxoset_size: block.extras.utxoSetSize ?? null,
coinbase_raw: block.extras.coinbaseRaw ?? null,
coinbase_address: block.extras.coinbaseAddress ?? null,
coinbase_signature: block.extras.coinbaseSignature ?? null,
coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
pool_slug: block.extras.pool.slug ?? null,
pool_id: block.extras.pool.id ?? null,
};
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
if (cleanBlock.fee_amt_percentiles === null) {
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
const summary = this.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
}
if (cleanBlock.fee_amt_percentiles !== null) {
cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
}
}
cleanBlock.fee_amt_percentiles = {
'min': cleanBlock.fee_amt_percentiles[0],
'perc_10': cleanBlock.fee_amt_percentiles[1],
'perc_25': cleanBlock.fee_amt_percentiles[2],
'perc_50': cleanBlock.fee_amt_percentiles[3],
'perc_75': cleanBlock.fee_amt_percentiles[4],
'perc_90': cleanBlock.fee_amt_percentiles[5],
'max': cleanBlock.fee_amt_percentiles[6],
};
cleanBlock.fee_rate_percentiles = {
'min': cleanBlock.fee_rate_percentiles[0],
'perc_10': cleanBlock.fee_rate_percentiles[1],
'perc_25': cleanBlock.fee_rate_percentiles[2],
'perc_50': cleanBlock.fee_rate_percentiles[3],
'perc_75': cleanBlock.fee_rate_percentiles[4],
'perc_90': cleanBlock.fee_rate_percentiles[5],
'max': cleanBlock.fee_rate_percentiles[6],
};
// Re-org can happen after indexing so we need to always get the
// latest state from core
cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
blocks.push(cleanBlock);
fromHeight++;
}
return blocks;
}
public async $getBlockAuditSummary(hash: string): Promise<any> {
let summary;
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {

61
backend/src/api/chain-tips.ts Normal file
View File

@@ -0,0 +1,61 @@
import logger from '../logger';
import bitcoinClient from './bitcoin/bitcoin-client';
export interface ChainTip {
height: number;
hash: string;
branchlen: number;
status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
};
export interface OrphanedBlock {
height: number;
hash: string;
status: 'valid-fork' | 'valid-headers' | 'headers-only';
}
class ChainTips {
private chainTips: ChainTip[] = [];
private orphanedBlocks: OrphanedBlock[] = [];
public async updateOrphanedBlocks(): Promise<void> {
try {
this.chainTips = await bitcoinClient.getChainTips();
this.orphanedBlocks = [];
for (const chain of this.chainTips) {
if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
let block = await bitcoinClient.getBlock(chain.hash);
while (block && block.confirmations === -1) {
this.orphanedBlocks.push({
height: block.height,
hash: block.hash,
status: chain.status
});
block = await bitcoinClient.getBlock(block.previousblockhash);
}
}
}
logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
} catch (e) {
logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
}
}
public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
if (height === undefined) {
return [];
}
const orphans: OrphanedBlock[] = [];
for (const block of this.orphanedBlocks) {
if (block.height === height) {
orphans.push(block);
}
}
return orphans;
}
}
export default new ChainTips();

15
backend/src/api/common.ts
View File

@@ -237,14 +237,21 @@ export class Common {
].join('x');
}
static utcDateToMysql(date?: number): string {
static utcDateToMysql(date?: number | null): string | null {
if (date === null) {
return null;
}
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
static findSocketNetwork(addr: string): {network: string | null, url: string} {
let network: string | null = null;
let url = addr.split('://')[1];
let url: string = addr;
if (config.LIGHTNING.BACKEND === 'cln') {
url = addr.split('://')[1];
}
if (!url) {
return {
@@ -261,7 +268,7 @@ export class Common {
}
} else if (addr.indexOf('i2p') !== -1) {
network = 'i2p';
} else if (addr.indexOf('ipv4') !== -1) {
} else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
const ipv = isIP(url.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
@@ -271,7 +278,7 @@ export class Common {
url: addr,
};
}
} else if (addr.indexOf('ipv6') !== -1) {
} else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
url = url.split('[')[1].split(']')[0];
const ipv = isIP(url);
if (ipv === 6) {

101
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 52;
private static currentVersion = 57;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -62,8 +62,8 @@ class DatabaseMigration {
if (databaseSchemaVersion <= 2) {
// Disable some spam logs when they're not relevant
this.uniqueLogs.push(this.blocksTruncatedMessage);
this.uniqueLogs.push(this.hashratesTruncatedMessage);
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
}
logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -86,7 +86,7 @@ class DatabaseMigration {
try {
await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
if (databaseSchemaVersion === 0) {
logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
} else {
logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
}
@@ -300,7 +300,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(27);
}
if (databaseSchemaVersion < 28 && isBitcoin === true) {
if (config.LIGHTNING.ENABLED) {
this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
@@ -464,10 +464,47 @@ class DatabaseMigration {
await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
await this.updateToSchemaVersion(52);
} catch(e) {
} catch (e) {
logger.warn('' + (e instanceof Error ? e.message : e));
}
}
if (databaseSchemaVersion < 53) {
await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
await this.updateToSchemaVersion(53);
}
if (databaseSchemaVersion < 54) {
this.uniqueLog(logger.notice, `'prices' table has been truncated`);
await this.$executeQuery(`TRUNCATE prices`);
if (isBitcoin === true) {
this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
await this.$executeQuery(`TRUNCATE blocks_prices`);
}
await this.updateToSchemaVersion(54);
}
if (databaseSchemaVersion < 55) {
await this.$executeQuery(this.getAdditionalBlocksDataQuery());
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.updateToSchemaVersion(55);
}
if (databaseSchemaVersion < 56) {
await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
await this.$executeQuery('TRUNCATE TABLE `blocks`');
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
this.uniqueLog(logger.notice, '`pools` table has been truncated`');
await this.updateToSchemaVersion(56);
}
if (databaseSchemaVersion < 57 && isBitcoin === true) {
await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
await this.updateToSchemaVersion(57);
}
}
/**
@@ -591,7 +628,7 @@ class DatabaseMigration {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
}
if (version < 9 && isBitcoin === true) {
if (version < 9 && isBitcoin === true) {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
}
@@ -741,6 +778,28 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getAdditionalBlocksDataQuery(): string {
return `ALTER TABLE blocks
ADD median_timestamp timestamp NOT NULL,
ADD coinbase_address varchar(100) NULL,
ADD coinbase_signature varchar(500) NULL,
ADD coinbase_signature_ascii varchar(500) NULL,
ADD avg_tx_size double unsigned NOT NULL,
ADD total_inputs int unsigned NOT NULL,
ADD total_outputs int unsigned NOT NULL,
ADD total_output_amt bigint unsigned NOT NULL,
ADD fee_percentiles longtext NULL,
ADD median_fee_amt int unsigned NULL,
ADD segwit_total_txs int unsigned NOT NULL,
ADD segwit_total_size int unsigned NOT NULL,
ADD segwit_total_weight int unsigned NOT NULL,
ADD header varchar(160) NOT NULL,
ADD utxoset_change int NOT NULL,
ADD utxoset_size int unsigned NULL,
ADD total_input_amt bigint unsigned NULL
`;
}
private getCreateDailyStatsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS hashrates (
hashrate_timestamp timestamp NOT NULL,
@@ -958,26 +1017,16 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];
public async $blocksReindexingTruncate(): Promise<void> {
logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
await Common.sleep$(5000);
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {
logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
continue;
}
await this.$executeQuery(`TRUNCATE ${table}`, true);
if (table === 'hashrates') {
await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
}
logger.notice(`Table ${table} has been truncated`);
}
} catch (e) {
logger.warn(`Unable to erase indexed data`);
}
}
await this.$executeQuery(`TRUNCATE blocks`);
await this.$executeQuery(`TRUNCATE hashrates`);
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
}
private async $convertCompactCpfpTables(): Promise<void> {
try {

21
backend/src/api/disk-cache.ts
View File

@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
import { Common } from './common';
class DiskCache {
private cacheSchemaVersion = 1;
private cacheSchemaVersion = 3;
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
@@ -62,9 +62,24 @@ class DiskCache {
}
wipeCache() {
fs.unlinkSync(DiskCache.FILE_NAME);
logger.notice(`Wipping nodejs backend cache/cache*.json files`);
try {
fs.unlinkSync(DiskCache.FILE_NAME);
} catch (e: any) {
if (e?.code !== 'ENOENT') {
logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
}
}
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
try {
fs.unlinkSync(filename);
} catch (e: any) {
if (e?.code !== 'ENOENT') {
logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
}
}
}
}

11
backend/src/api/explorer/channels.api.ts
View File

@@ -559,6 +559,17 @@ class ChannelsApi {
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
// https://github.com/mempool/mempool/issues/3006
if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
channel.last_update = null;
}
if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
policy1.last_update = null;
}
if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
policy2.last_update = null;
}
const query = `INSERT INTO channels
(
id,

33
backend/src/api/explorer/nodes.api.ts
View File

@@ -228,7 +228,7 @@ class NodesApi {
nodes.capacity
FROM nodes
ORDER BY capacity DESC
LIMIT 100
LIMIT 6
`;
[rows] = await DB.query(query);
@@ -269,14 +269,26 @@ class NodesApi {
let query: string;
if (full === false) {
query = `
SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
nodes.channels
SELECT
nodes.public_key as publicKey,
IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
nodes.channels,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
ORDER BY channels DESC
LIMIT 100;
LIMIT 6;
`;
[rows] = await DB.query(query);
for (let i = 0; i < rows.length; ++i) {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);
}
} else {
query = `
SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
@@ -362,7 +374,13 @@ class NodesApi {
public async $searchNodeByPublicKeyOrAlias(search: string) {
try {
const publicKeySearch = search.replace('%', '') + '%';
const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
const aliasSearch = search
.replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".
.replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
.split(' ')
.filter(key => key.length)
.map((search) => '+' + search + '*').join(' ');
// %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
return rows;
@@ -624,6 +642,11 @@ class NodesApi {
*/
public async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
// https://github.com/mempool/mempool/issues/3006
if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
node.last_update = null;
}
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,

6
backend/src/api/lightning/lightning-api.interface.ts
View File

@@ -21,7 +21,7 @@ export namespace ILightningApi {
export interface Channel {
channel_id: string;
chan_point: string;
last_update: number;
last_update: number | null;
node1_pub: string;
node2_pub: string;
capacity: string;
@@ -36,11 +36,11 @@ export namespace ILightningApi {
fee_rate_milli_msat: string;
disabled: boolean;
max_htlc_msat: string;
last_update: number;
last_update: number | null;
}
export interface Node {
last_update: number;
last_update: number | null;
pub_key: string;
alias: string;
addresses: {

20
backend/src/api/mining/mining-routes.ts
View File

@@ -1,13 +1,13 @@
import { Application, Request, Response } from 'express';
import config from "../../config";
import logger from '../../logger';
import audits from '../audit';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import BlocksRepository from '../../repositories/BlocksRepository';
import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
import HashratesRepository from '../../repositories/HashratesRepository';
import bitcoinClient from '../bitcoin/bitcoin-client';
import mining from "./mining";
import PricesRepository from '../../repositories/PricesRepository';
class MiningRoutes {
public initRoutes(app: Application) {
@@ -32,9 +32,27 @@ class MiningRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
.get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
;
}
private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
try {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
if (req.query.timestamp) {
res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
parseInt(<string>req.query.timestamp ?? 0, 10)
));
} else {
res.status(200).send(await PricesRepository.$getHistoricalPrices());
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getPool(req: Request, res: Response): Promise<void> {
try {
const stats = await mining.$getPoolStat(req.params.slug);

56
backend/src/api/mining/mining.ts
View File

@@ -11,6 +11,8 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
import config from '../../config';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository';
import bitcoinApiFactory from '../bitcoin/bitcoin-api-factory';
import { IEsploraApi } from '../bitcoin/esplora-api.interface';
class Mining {
blocksPriceIndexingRunning = false;
@@ -100,6 +102,7 @@ class Mining {
rank: rank++,
emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
slug: poolInfo.slug,
avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
};
poolsStats.push(poolStat);
});
@@ -171,7 +174,7 @@ class Mining {
}
/**
* [INDEXING] Generate weekly mining pool hashrate history
* Generate weekly mining pool hashrate history
*/
public async $generatePoolHashrateHistory(): Promise<void> {
const now = new Date();
@@ -188,8 +191,8 @@ class Mining {
try {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.time * 1000;
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
const hashrates: any[] = [];
@@ -278,7 +281,7 @@ class Mining {
}
/**
* [INDEXING] Generate daily hashrate data
* Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// We only run this once a day around midnight
@@ -291,8 +294,8 @@ class Mining {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
try {
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.time * 1000;
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round(lastMidnight.getTime());
@@ -393,13 +396,13 @@ class Mining {
}
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
let currentDifficulty = genesisBlock.difficulty;
let totalIndexed = 0;
if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: genesisBlock.time,
time: genesisBlock.timestamp,
height: 0,
difficulty: currentDifficulty,
adjustment: 0.0,
@@ -458,7 +461,7 @@ class Mining {
/**
* Create a link between blocks and the latest price at when they were mined
*/
public async $indexBlockPrices() {
public async $indexBlockPrices(): Promise<void> {
if (this.blocksPriceIndexingRunning === true) {
return;
}
@@ -519,6 +522,41 @@ class Mining {
this.blocksPriceIndexingRunning = false;
}
/**
* Index core coinstatsindex
*/
public async $indexCoinStatsIndex(): Promise<void> {
let timer = new Date().getTime() / 1000;
let totalIndexed = 0;
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
let currentBlockHeight = blockchainInfo.blocks;
while (currentBlockHeight > 0) {
const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
currentBlockHeight, currentBlockHeight - 10000);
for (const block of indexedBlocks) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
Math.round(txoutset.block_info.prevout_spent * 100000000));
++totalIndexed;
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
if (elapsedSeconds > 5) {
logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
timer = new Date().getTime() / 1000;
}
}
currentBlockHeight -= 10000;
}
if (totalIndexed) {
logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
}
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);

336
backend/src/api/pools-parser.ts
View File

@@ -1,289 +1,161 @@
import DB from '../database';
import logger from '../logger';
import config from '../config';
import BlocksRepository from '../repositories/BlocksRepository';
interface Pool {
name: string;
link: string;
regexes: string[];
addresses: string[];
slug: string;
}
import PoolsRepository from '../repositories/PoolsRepository';
import { PoolTag } from '../mempool.interfaces';
import diskCache from './disk-cache';
class PoolsParser {
miningPools: any[] = [];
unknownPool: any = {
'id': 0,
'name': 'Unknown',
'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
'regexes': '[]',
'addresses': '[]',
'slug': 'unknown'
};
slugWarnFlag = false;
private uniqueLogs: string[] = [];
private uniqueLog(loggerFunction: any, msg: string): void {
if (this.uniqueLogs.includes(msg)) {
return;
}
this.uniqueLogs.push(msg);
loggerFunction(msg);
}
public setMiningPools(pools): void {
for (const pool of pools) {
pool.regexes = pool.tags;
pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
delete(pool.tags);
}
this.miningPools = pools;
}
/**
* Parse the pools.json file, consolidate the data and dump it into the database
* Populate our db with updated mining pool definition
* @param pools
*/
public async migratePoolsJson(poolsJson: object): Promise<void> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
public async migratePoolsJson(): Promise<void> {
// We also need to wipe the backend cache to make sure we don't serve blocks with
// the wrong mining pool (usually happen with unknown blocks)
diskCache.wipeCache();
// First we save every entries without paying attention to pool duplication
const poolsDuplicated: Pool[] = [];
await this.$insertUnknownPool();
const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
for (let i = 0; i < coinbaseTags.length; ++i) {
poolsDuplicated.push({
'name': (<Pool>coinbaseTags[i][1]).name,
'link': (<Pool>coinbaseTags[i][1]).link,
'regexes': [coinbaseTags[i][0]],
'addresses': [],
'slug': ''
});
}
const addressesTags = Object.entries(poolsJson['payout_addresses']);
for (let i = 0; i < addressesTags.length; ++i) {
poolsDuplicated.push({
'name': (<Pool>addressesTags[i][1]).name,
'link': (<Pool>addressesTags[i][1]).link,
'regexes': [],
'addresses': [addressesTags[i][0]],
'slug': ''
});
}
// Then, we find unique mining pool names
const poolNames: string[] = [];
for (let i = 0; i < poolsDuplicated.length; ++i) {
if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
poolNames.push(poolsDuplicated[i].name);
for (const pool of this.miningPools) {
if (!pool.id) {
logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
continue;
}
}
logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);
// Get existing pools from the db
let existingPools;
try {
if (config.DATABASE.ENABLED === true) {
[existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
if (!poolDB) {
// New mining pool
const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.debug(`Inserting new mining pool ${pool.name}`);
await PoolsRepository.$insertNewMiningPool(pool, slug);
await this.$deleteUnknownBlocks();
} else {
existingPools = [];
}
} catch (e) {
logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
return;
}
this.miningPools = [];
// Finally, we generate the final consolidated pools data
const finalPoolDataAdd: Pool[] = [];
const finalPoolDataUpdate: Pool[] = [];
const finalPoolDataRename: Pool[] = [];
for (let i = 0; i < poolNames.length; ++i) {
let allAddresses: string[] = [];
let allRegexes: string[] = [];
const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
for (let y = 0; y < match.length; ++y) {
allAddresses = allAddresses.concat(match[y].addresses);
allRegexes = allRegexes.concat(match[y].regexes);
}
const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
let slug: string | undefined;
try {
slug = poolsJson['slugs'][poolNames[i]];
} catch (e) {
if (this.slugWarnFlag === false) {
logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
this.slugWarnFlag = true;
if (poolDB.name !== pool.name) {
// Pool has been renamed
const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
}
}
if (slug === undefined) {
// Only keep alphanumerical
slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
}
const poolObj = {
'name': finalPoolName,
'link': match[0].link,
'regexes': allRegexes,
'addresses': allAddresses,
'slug': slug
};
const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
if (existingPool !== undefined) {
// Check if any data was actually updated
const equals = (a, b) =>
a.length === b.length &&
a.every((v, i) => v === b[i]);
if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
finalPoolDataUpdate.push(poolObj);
if (poolDB.link !== pool.link) {
// Pool link has changed
logger.debug(`Updating link for ${pool.name} mining pool`);
await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
}
} else if (config.DATABASE.ENABLED) {
// Double check that if we're not just renaming a pool (same address same regex)
const [poolToRename]: any[] = await DB.query(`
SELECT * FROM pools
WHERE addresses = ? OR regexes = ?`,
[JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
);
if (poolToRename && poolToRename.length > 0) {
// We're actually renaming an existing pool
finalPoolDataRename.push({
'name': poolObj.name,
'link': poolObj.link,
'regexes': allRegexes,
'addresses': allAddresses,
'slug': slug
});
logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
} else {
logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
finalPoolDataAdd.push(poolObj);
if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
JSON.stringify(pool.regexes) !== poolDB.regexes) {
// Pool addresses changed or coinbase tags changed
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
await this.$deleteBlocksForPool(poolDB);
}
}
this.miningPools.push({
'name': finalPoolName,
'link': match[0].link,
'regexes': JSON.stringify(allRegexes),
'addresses': JSON.stringify(allAddresses),
'slug': slug
});
}
if (config.DATABASE.ENABLED === false) { // Don't run db operations
logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
return;
}
if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
finalPoolDataRename.length > 0
) {
logger.debug(`Update pools table now`, logger.tags.mining);
// Add new mining pools into the database
let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
for (let i = 0; i < finalPoolDataAdd.length; ++i) {
queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
'${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
}
queryAdd = queryAdd.slice(0, -1) + ';';
// Updated existing mining pools in the database
const updateQueries: string[] = [];
for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
updateQueries.push(`
UPDATE pools
SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
slug='${finalPoolDataUpdate[i].slug}'
WHERE name='${finalPoolDataUpdate[i].name}'
;`);
}
// Rename mining pools
const renameQueries: string[] = [];
for (let i = 0; i < finalPoolDataRename.length; ++i) {
renameQueries.push(`
UPDATE pools
SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
slug='${finalPoolDataRename[i].slug}'
WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
;`);
}
try {
if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
await this.$deleteBlocskToReindex(finalPoolDataUpdate);
}
if (finalPoolDataAdd.length > 0) {
await DB.query({ sql: queryAdd, timeout: 120000 });
}
for (const query of updateQueries) {
await DB.query({ sql: query, timeout: 120000 });
}
for (const query of renameQueries) {
await DB.query({ sql: query, timeout: 120000 });
}
await this.insertUnknownPool();
logger.info('Mining pools.json import completed', logger.tags.mining);
} catch (e) {
logger.err(`Cannot import pools in the database`, logger.tags.mining);
throw e;
}
}
try {
await this.insertUnknownPool();
} catch (e) {
logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
throw e;
}
logger.info('Mining pools-v2.json import completed');
}
/**
* Manually add the 'unknown pool'
*/
private async insertUnknownPool() {
public async $insertUnknownPool(): Promise<void> {
if (!config.DATABASE.ENABLED) {
return;
}
try {
const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
if (rows.length === 0) {
await DB.query({
sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
`});
} else {
await DB.query(`UPDATE pools
SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
regexes='[]', addresses='[]',
slug='unknown'
WHERE name='Unknown'
slug='${this.unknownPool.slug}',
unique_id=0
WHERE slug='${this.unknownPool.slug}'
`);
}
} catch (e) {
logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
}
}
/**
* Delete blocks which needs to be reindexed
* Delete indexed blocks for an updated mining pool
*
* @param pool
*/
private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
return;
}
const blockCount = await BlocksRepository.$blockCount(null, null);
if (blockCount === 0) {
return;
}
for (const updatedPool of finalPoolDataUpdate) {
const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
if (pool.length > 0) {
logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
}
}
// Ignore early days of Bitcoin as there were not mining pool yet
logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
// Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
// Ignore early days of Bitcoin as there were no mining pool yet
const [oldestPoolBlock]: any[] = await DB.query(`
SELECT height
FROM blocks
WHERE pool_id = ?
ORDER BY height
LIMIT 1`,
[pool.id]
);
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
[unknownPool[0].id]
);
logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ?`,
[pool.id]
);
}
logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
await DB.query(`DELETE FROM hashrates`);
private async $deleteUnknownBlocks(): Promise<void> {
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ? AND height >= 130635`,
[unknownPool[0].id]
);
}
}

1
backend/src/api/transaction-utils.ts
View File

@@ -14,6 +14,7 @@ class TransactionUtils {
vout: tx.vout
.map((vout) => ({
scriptpubkey_address: vout.scriptpubkey_address,
scriptpubkey_asm: vout.scriptpubkey_asm,
value: vout.value
}))
.filter((vout) => vout.value)