Merge branch 'master' into nymkappa/feature/align-dashboards
This commit is contained in:
wiz
GitHub
@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
||||
size: block.size,
|
||||
weight: block.weight,
|
||||
previousblockhash: block.previousblockhash,
|
||||
medianTime: block.mediantime,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -95,6 +95,8 @@ class BitcoinRoutes {
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
|
||||
.post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
|
||||
;
|
||||
|
||||
if (config.MEMPOOL.BACKEND !== 'esplora') {
|
||||
@@ -402,6 +404,41 @@ class BitcoinRoutes {
|
||||
}
|
||||
}
|
||||
|
||||
private async getBlocksByBulk(req: Request, res: Response) {
|
||||
try {
|
||||
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
|
||||
return res.status(404).send(`This API is only available for Bitcoin networks`);
|
||||
}
|
||||
if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
|
||||
return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
|
||||
}
|
||||
if (!Common.indexingEnabled()) {
|
||||
return res.status(404).send(`Indexing is required for this API`);
|
||||
}
|
||||
|
||||
const from = parseInt(req.params.from, 10);
|
||||
if (!req.params.from || from < 0) {
|
||||
return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
|
||||
}
|
||||
const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
|
||||
if (to < 0) {
|
||||
return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
|
||||
}
|
||||
if (from > to) {
|
||||
return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
|
||||
}
|
||||
if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
|
||||
return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
|
||||
}
|
||||
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
res.json(await blocks.$getBlocksBetweenHeight(from, to));
|
||||
|
||||
} catch (e) {
|
||||
res.status(500).send(e instanceof Error ? e.message : e);
|
||||
}
|
||||
}
|
||||
|
||||
private async getLegacyBlocks(req: Request, res: Response) {
|
||||
try {
|
||||
const returnBlocks: IEsploraApi.Block[] = [];
|
||||
|
||||
@@ -88,6 +88,7 @@ export namespace IEsploraApi {
|
||||
size: number;
|
||||
weight: number;
|
||||
previousblockhash: string;
|
||||
medianTime?: number;
|
||||
}
|
||||
|
||||
export interface Address {
|
||||
|
||||
@@ -25,6 +25,7 @@ import mining from './mining/mining';
|
||||
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
|
||||
import PricesRepository from '../repositories/PricesRepository';
|
||||
import priceUpdater from '../tasks/price-updater';
|
||||
import chainTips from './chain-tips';
|
||||
|
||||
class Blocks {
|
||||
private blocks: BlockExtended[] = [];
|
||||
@@ -165,33 +166,80 @@ class Blocks {
|
||||
* @returns BlockExtended
|
||||
*/
|
||||
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
|
||||
const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
|
||||
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
|
||||
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
|
||||
blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
|
||||
blockExtended.extras.usd = priceUpdater.latestPrices.USD;
|
||||
const blk: BlockExtended = Object.assign({ extras: {} }, block);
|
||||
blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
|
||||
blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
|
||||
blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
|
||||
blk.extras.usd = priceUpdater.latestPrices.USD;
|
||||
blk.extras.medianTimestamp = block.medianTime;
|
||||
blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
|
||||
|
||||
if (block.height === 0) {
|
||||
blockExtended.extras.medianFee = 0; // 50th percentiles
|
||||
blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
|
||||
blockExtended.extras.totalFees = 0;
|
||||
blockExtended.extras.avgFee = 0;
|
||||
blockExtended.extras.avgFeeRate = 0;
|
||||
blk.extras.medianFee = 0; // 50th percentiles
|
||||
blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
|
||||
blk.extras.totalFees = 0;
|
||||
blk.extras.avgFee = 0;
|
||||
blk.extras.avgFeeRate = 0;
|
||||
blk.extras.utxoSetChange = 0;
|
||||
blk.extras.avgTxSize = 0;
|
||||
blk.extras.totalInputs = 0;
|
||||
blk.extras.totalOutputs = 1;
|
||||
blk.extras.totalOutputAmt = 0;
|
||||
blk.extras.segwitTotalTxs = 0;
|
||||
blk.extras.segwitTotalSize = 0;
|
||||
blk.extras.segwitTotalWeight = 0;
|
||||
} else {
|
||||
const stats = await bitcoinClient.getBlockStats(block.id, [
|
||||
'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
|
||||
]);
|
||||
blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
|
||||
blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
|
||||
blockExtended.extras.totalFees = stats.totalfee;
|
||||
blockExtended.extras.avgFee = stats.avgfee;
|
||||
blockExtended.extras.avgFeeRate = stats.avgfeerate;
|
||||
const stats = await bitcoinClient.getBlockStats(block.id);
|
||||
blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
|
||||
blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
|
||||
blk.extras.totalFees = stats.totalfee;
|
||||
blk.extras.avgFee = stats.avgfee;
|
||||
blk.extras.avgFeeRate = stats.avgfeerate;
|
||||
blk.extras.utxoSetChange = stats.utxo_increase;
|
||||
blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
|
||||
blk.extras.totalInputs = stats.ins;
|
||||
blk.extras.totalOutputs = stats.outs;
|
||||
blk.extras.totalOutputAmt = stats.total_out;
|
||||
blk.extras.segwitTotalTxs = stats.swtxs;
|
||||
blk.extras.segwitTotalSize = stats.swtotal_size;
|
||||
blk.extras.segwitTotalWeight = stats.swtotal_weight;
|
||||
}
|
||||
|
||||
if (Common.blocksSummariesIndexingEnabled()) {
|
||||
blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
|
||||
if (blk.extras.feePercentiles !== null) {
|
||||
blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
|
||||
}
|
||||
}
|
||||
|
||||
blk.extras.virtualSize = block.weight / 4.0;
|
||||
if (blk.extras.coinbaseTx.vout.length > 0) {
|
||||
blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
|
||||
blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
|
||||
blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null;
|
||||
} else {
|
||||
blk.extras.coinbaseAddress = null;
|
||||
blk.extras.coinbaseSignature = null;
|
||||
blk.extras.coinbaseSignatureAscii = null;
|
||||
}
|
||||
|
||||
const header = await bitcoinClient.getBlockHeader(block.id, false);
|
||||
blk.extras.header = header;
|
||||
|
||||
const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
|
||||
if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
|
||||
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
|
||||
blk.extras.utxoSetSize = txoutset.txouts,
|
||||
blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
|
||||
} else {
|
||||
blk.extras.utxoSetSize = null;
|
||||
blk.extras.totalInputAmt = null;
|
||||
}
|
||||
|
||||
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
|
||||
let pool: PoolTag;
|
||||
if (blockExtended.extras?.coinbaseTx !== undefined) {
|
||||
pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
|
||||
if (blk.extras?.coinbaseTx !== undefined) {
|
||||
pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
|
||||
} else {
|
||||
if (config.DATABASE.ENABLED === true) {
|
||||
pool = await poolsRepository.$getUnknownPool();
|
||||
@@ -201,10 +249,10 @@ class Blocks {
|
||||
}
|
||||
|
||||
if (!pool) { // We should never have this situation in practise
|
||||
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
|
||||
logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
|
||||
`Check your "pools" table entries`);
|
||||
} else {
|
||||
blockExtended.extras.pool = {
|
||||
blk.extras.pool = {
|
||||
id: pool.id,
|
||||
name: pool.name,
|
||||
slug: pool.slug,
|
||||
@@ -214,12 +262,12 @@ class Blocks {
|
||||
if (config.MEMPOOL.AUDIT) {
|
||||
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
|
||||
if (auditScore != null) {
|
||||
blockExtended.extras.matchRate = auditScore.matchRate;
|
||||
blk.extras.matchRate = auditScore.matchRate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return blockExtended;
|
||||
return blk;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -500,6 +548,7 @@ class Blocks {
|
||||
} else {
|
||||
this.currentBlockHeight++;
|
||||
logger.debug(`New block found (#${this.currentBlockHeight})!`);
|
||||
await chainTips.updateOrphanedBlocks();
|
||||
}
|
||||
|
||||
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
|
||||
@@ -688,7 +737,6 @@ class Blocks {
|
||||
}
|
||||
|
||||
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
|
||||
|
||||
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
|
||||
if (currentHeight > this.currentBlockHeight) {
|
||||
limit -= currentHeight - this.currentBlockHeight;
|
||||
@@ -728,6 +776,113 @@ class Blocks {
|
||||
return returnBlocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used for bulk block data query
|
||||
*
|
||||
* @param fromHeight
|
||||
* @param toHeight
|
||||
*/
|
||||
public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
|
||||
if (!Common.indexingEnabled()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const blocks: any[] = [];
|
||||
|
||||
while (fromHeight <= toHeight) {
|
||||
let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
|
||||
if (!block) {
|
||||
await this.$indexBlock(fromHeight);
|
||||
block = await blocksRepository.$getBlockByHeight(fromHeight);
|
||||
if (!block) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup fields before sending the response
|
||||
const cleanBlock: any = {
|
||||
height: block.height ?? null,
|
||||
hash: block.id ?? null,
|
||||
timestamp: block.blockTimestamp ?? null,
|
||||
median_timestamp: block.medianTime ?? null,
|
||||
previous_block_hash: block.previousblockhash ?? null,
|
||||
difficulty: block.difficulty ?? null,
|
||||
header: block.header ?? null,
|
||||
version: block.version ?? null,
|
||||
bits: block.bits ?? null,
|
||||
nonce: block.nonce ?? null,
|
||||
size: block.size ?? null,
|
||||
weight: block.weight ?? null,
|
||||
tx_count: block.tx_count ?? null,
|
||||
merkle_root: block.merkle_root ?? null,
|
||||
reward: block.reward ?? null,
|
||||
total_fee_amt: block.fees ?? null,
|
||||
avg_fee_amt: block.avg_fee ?? null,
|
||||
median_fee_amt: block.median_fee_amt ?? null,
|
||||
fee_amt_percentiles: block.fee_percentiles ?? null,
|
||||
avg_fee_rate: block.avg_fee_rate ?? null,
|
||||
median_fee_rate: block.median_fee ?? null,
|
||||
fee_rate_percentiles: block.fee_span ?? null,
|
||||
total_inputs: block.total_inputs ?? null,
|
||||
total_input_amt: block.total_input_amt ?? null,
|
||||
total_outputs: block.total_outputs ?? null,
|
||||
total_output_amt: block.total_output_amt ?? null,
|
||||
segwit_total_txs: block.segwit_total_txs ?? null,
|
||||
segwit_total_size: block.segwit_total_size ?? null,
|
||||
segwit_total_weight: block.segwit_total_weight ?? null,
|
||||
avg_tx_size: block.avg_tx_size ?? null,
|
||||
utxoset_change: block.utxoset_change ?? null,
|
||||
utxoset_size: block.utxoset_size ?? null,
|
||||
coinbase_raw: block.coinbase_raw ?? null,
|
||||
coinbase_address: block.coinbase_address ?? null,
|
||||
coinbase_signature: block.coinbase_signature ?? null,
|
||||
coinbase_signature_ascii: block.coinbase_signature_ascii ?? null,
|
||||
pool_slug: block.pool_slug ?? null,
|
||||
};
|
||||
|
||||
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
|
||||
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
|
||||
if (cleanBlock.fee_amt_percentiles === null) {
|
||||
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
|
||||
const summary = this.summarizeBlock(block);
|
||||
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
|
||||
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
|
||||
}
|
||||
if (cleanBlock.fee_amt_percentiles !== null) {
|
||||
cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
|
||||
}
|
||||
}
|
||||
|
||||
cleanBlock.fee_amt_percentiles = {
|
||||
'min': cleanBlock.fee_amt_percentiles[0],
|
||||
'perc_10': cleanBlock.fee_amt_percentiles[1],
|
||||
'perc_25': cleanBlock.fee_amt_percentiles[2],
|
||||
'perc_50': cleanBlock.fee_amt_percentiles[3],
|
||||
'perc_75': cleanBlock.fee_amt_percentiles[4],
|
||||
'perc_90': cleanBlock.fee_amt_percentiles[5],
|
||||
'max': cleanBlock.fee_amt_percentiles[6],
|
||||
};
|
||||
cleanBlock.fee_rate_percentiles = {
|
||||
'min': cleanBlock.fee_rate_percentiles[0],
|
||||
'perc_10': cleanBlock.fee_rate_percentiles[1],
|
||||
'perc_25': cleanBlock.fee_rate_percentiles[2],
|
||||
'perc_50': cleanBlock.fee_rate_percentiles[3],
|
||||
'perc_75': cleanBlock.fee_rate_percentiles[4],
|
||||
'perc_90': cleanBlock.fee_rate_percentiles[5],
|
||||
'max': cleanBlock.fee_rate_percentiles[6],
|
||||
};
|
||||
|
||||
// Re-org can happen after indexing so we need to always get the
|
||||
// latest state from core
|
||||
cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
|
||||
|
||||
blocks.push(cleanBlock);
|
||||
fromHeight++;
|
||||
}
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
public async $getBlockAuditSummary(hash: string): Promise<any> {
|
||||
let summary;
|
||||
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
|
||||
|
||||
57
backend/src/api/chain-tips.ts
Normal file
57
backend/src/api/chain-tips.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
import logger from "../logger";
|
||||
import bitcoinClient from "./bitcoin/bitcoin-client";
|
||||
|
||||
export interface ChainTip {
|
||||
height: number;
|
||||
hash: string;
|
||||
branchlen: number;
|
||||
status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
|
||||
};
|
||||
|
||||
export interface OrphanedBlock {
|
||||
height: number;
|
||||
hash: string;
|
||||
status: 'valid-fork' | 'valid-headers' | 'headers-only';
|
||||
}
|
||||
|
||||
class ChainTips {
|
||||
private chainTips: ChainTip[] = [];
|
||||
private orphanedBlocks: OrphanedBlock[] = [];
|
||||
|
||||
public async updateOrphanedBlocks(): Promise<void> {
|
||||
try {
|
||||
this.chainTips = await bitcoinClient.getChainTips();
|
||||
this.orphanedBlocks = [];
|
||||
|
||||
for (const chain of this.chainTips) {
|
||||
if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
|
||||
let block = await bitcoinClient.getBlock(chain.hash);
|
||||
while (block && block.confirmations === -1) {
|
||||
this.orphanedBlocks.push({
|
||||
height: block.height,
|
||||
hash: block.hash,
|
||||
status: chain.status
|
||||
});
|
||||
block = await bitcoinClient.getBlock(block.previousblockhash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
|
||||
} catch (e) {
|
||||
logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
|
||||
public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {
|
||||
const orphans: OrphanedBlock[] = [];
|
||||
for (const block of this.orphanedBlocks) {
|
||||
if (block.height === height) {
|
||||
orphans.push(block);
|
||||
}
|
||||
}
|
||||
return orphans;
|
||||
}
|
||||
}
|
||||
|
||||
export default new ChainTips();
|
||||
@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
|
||||
import { RowDataPacket } from 'mysql2';
|
||||
|
||||
class DatabaseMigration {
|
||||
private static currentVersion = 54;
|
||||
private static currentVersion = 56;
|
||||
private queryTimeout = 3600_000;
|
||||
private statisticsAddedIndexed = false;
|
||||
private uniqueLogs: string[] = [];
|
||||
@@ -62,8 +62,8 @@ class DatabaseMigration {
|
||||
|
||||
if (databaseSchemaVersion <= 2) {
|
||||
// Disable some spam logs when they're not relevant
|
||||
this.uniqueLogs.push(this.blocksTruncatedMessage);
|
||||
this.uniqueLogs.push(this.hashratesTruncatedMessage);
|
||||
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
|
||||
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
|
||||
}
|
||||
|
||||
logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
|
||||
@@ -483,6 +483,23 @@ class DatabaseMigration {
|
||||
}
|
||||
await this.updateToSchemaVersion(54);
|
||||
}
|
||||
|
||||
if (databaseSchemaVersion < 55) {
|
||||
await this.$executeQuery(this.getAdditionalBlocksDataQuery());
|
||||
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
|
||||
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
|
||||
await this.updateToSchemaVersion(55);
|
||||
}
|
||||
|
||||
if (databaseSchemaVersion < 56) {
|
||||
await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
|
||||
await this.$executeQuery('TRUNCATE TABLE `blocks`');
|
||||
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
|
||||
await this.$executeQuery('DELETE FROM `pools`');
|
||||
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
|
||||
this.uniqueLog(logger.notice, '`pools` table has been truncated`');
|
||||
await this.updateToSchemaVersion(56);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -756,6 +773,28 @@ class DatabaseMigration {
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
|
||||
}
|
||||
|
||||
private getAdditionalBlocksDataQuery(): string {
|
||||
return `ALTER TABLE blocks
|
||||
ADD median_timestamp timestamp NOT NULL,
|
||||
ADD coinbase_address varchar(100) NULL,
|
||||
ADD coinbase_signature varchar(500) NULL,
|
||||
ADD coinbase_signature_ascii varchar(500) NULL,
|
||||
ADD avg_tx_size double unsigned NOT NULL,
|
||||
ADD total_inputs int unsigned NOT NULL,
|
||||
ADD total_outputs int unsigned NOT NULL,
|
||||
ADD total_output_amt bigint unsigned NOT NULL,
|
||||
ADD fee_percentiles longtext NULL,
|
||||
ADD median_fee_amt int unsigned NULL,
|
||||
ADD segwit_total_txs int unsigned NOT NULL,
|
||||
ADD segwit_total_size int unsigned NOT NULL,
|
||||
ADD segwit_total_weight int unsigned NOT NULL,
|
||||
ADD header varchar(160) NOT NULL,
|
||||
ADD utxoset_change int NOT NULL,
|
||||
ADD utxoset_size int unsigned NULL,
|
||||
ADD total_input_amt bigint unsigned NULL
|
||||
`;
|
||||
}
|
||||
|
||||
private getCreateDailyStatsTableQuery(): string {
|
||||
return `CREATE TABLE IF NOT EXISTS hashrates (
|
||||
hashrate_timestamp timestamp NOT NULL,
|
||||
|
||||
@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
|
||||
import { Common } from './common';
|
||||
|
||||
class DiskCache {
|
||||
private cacheSchemaVersion = 1;
|
||||
private cacheSchemaVersion = 2;
|
||||
|
||||
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
|
||||
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
|
||||
|
||||
@@ -172,7 +172,7 @@ class Mining {
|
||||
}
|
||||
|
||||
/**
|
||||
* [INDEXING] Generate weekly mining pool hashrate history
|
||||
* Generate weekly mining pool hashrate history
|
||||
*/
|
||||
public async $generatePoolHashrateHistory(): Promise<void> {
|
||||
const now = new Date();
|
||||
@@ -279,7 +279,7 @@ class Mining {
|
||||
}
|
||||
|
||||
/**
|
||||
* [INDEXING] Generate daily hashrate data
|
||||
* Generate daily hashrate data
|
||||
*/
|
||||
public async $generateNetworkHashrateHistory(): Promise<void> {
|
||||
// We only run this once a day around midnight
|
||||
@@ -459,7 +459,7 @@ class Mining {
|
||||
/**
|
||||
* Create a link between blocks and the latest price at when they were mined
|
||||
*/
|
||||
public async $indexBlockPrices() {
|
||||
public async $indexBlockPrices(): Promise<void> {
|
||||
if (this.blocksPriceIndexingRunning === true) {
|
||||
return;
|
||||
}
|
||||
@@ -520,6 +520,41 @@ class Mining {
|
||||
this.blocksPriceIndexingRunning = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index core coinstatsindex
|
||||
*/
|
||||
public async $indexCoinStatsIndex(): Promise<void> {
|
||||
let timer = new Date().getTime() / 1000;
|
||||
let totalIndexed = 0;
|
||||
|
||||
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
|
||||
let currentBlockHeight = blockchainInfo.blocks;
|
||||
|
||||
while (currentBlockHeight > 0) {
|
||||
const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
|
||||
currentBlockHeight, currentBlockHeight - 10000);
|
||||
|
||||
for (const block of indexedBlocks) {
|
||||
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
|
||||
await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
|
||||
Math.round(txoutset.block_info.prevout_spent * 100000000));
|
||||
++totalIndexed;
|
||||
|
||||
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
|
||||
if (elapsedSeconds > 5) {
|
||||
logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
|
||||
timer = new Date().getTime() / 1000;
|
||||
}
|
||||
}
|
||||
|
||||
currentBlockHeight -= 10000;
|
||||
}
|
||||
|
||||
if (totalIndexed) {
|
||||
logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
|
||||
}
|
||||
}
|
||||
|
||||
private getDateMidnight(date: Date): Date {
|
||||
date.setUTCHours(0);
|
||||
date.setUTCMinutes(0);
|
||||
|
||||
@@ -1,15 +1,8 @@
|
||||
import DB from '../database';
|
||||
import logger from '../logger';
|
||||
import config from '../config';
|
||||
import BlocksRepository from '../repositories/BlocksRepository';
|
||||
|
||||
interface Pool {
|
||||
name: string;
|
||||
link: string;
|
||||
regexes: string[];
|
||||
addresses: string[];
|
||||
slug: string;
|
||||
}
|
||||
import PoolsRepository from '../repositories/PoolsRepository';
|
||||
import { PoolTag } from '../mempool.interfaces';
|
||||
|
||||
class PoolsParser {
|
||||
miningPools: any[] = [];
|
||||
@@ -20,270 +13,142 @@ class PoolsParser {
|
||||
'addresses': '[]',
|
||||
'slug': 'unknown'
|
||||
};
|
||||
slugWarnFlag = false;
|
||||
private uniqueLogs: string[] = [];
|
||||
|
||||
private uniqueLog(loggerFunction: any, msg: string): void {
|
||||
if (this.uniqueLogs.includes(msg)) {
|
||||
return;
|
||||
}
|
||||
this.uniqueLogs.push(msg);
|
||||
loggerFunction(msg);
|
||||
}
|
||||
|
||||
public setMiningPools(pools): void {
|
||||
for (const pool of pools) {
|
||||
pool.regexes = pool.tags;
|
||||
delete(pool.tags);
|
||||
}
|
||||
this.miningPools = pools;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the pools.json file, consolidate the data and dump it into the database
|
||||
* Populate our db with updated mining pool definition
|
||||
* @param pools
|
||||
*/
|
||||
public async migratePoolsJson(poolsJson: object): Promise<void> {
|
||||
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
|
||||
return;
|
||||
}
|
||||
public async migratePoolsJson(): Promise<void> {
|
||||
await this.$insertUnknownPool();
|
||||
|
||||
// First we save every entries without paying attention to pool duplication
|
||||
const poolsDuplicated: Pool[] = [];
|
||||
|
||||
const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
|
||||
for (let i = 0; i < coinbaseTags.length; ++i) {
|
||||
poolsDuplicated.push({
|
||||
'name': (<Pool>coinbaseTags[i][1]).name,
|
||||
'link': (<Pool>coinbaseTags[i][1]).link,
|
||||
'regexes': [coinbaseTags[i][0]],
|
||||
'addresses': [],
|
||||
'slug': ''
|
||||
});
|
||||
}
|
||||
const addressesTags = Object.entries(poolsJson['payout_addresses']);
|
||||
for (let i = 0; i < addressesTags.length; ++i) {
|
||||
poolsDuplicated.push({
|
||||
'name': (<Pool>addressesTags[i][1]).name,
|
||||
'link': (<Pool>addressesTags[i][1]).link,
|
||||
'regexes': [],
|
||||
'addresses': [addressesTags[i][0]],
|
||||
'slug': ''
|
||||
});
|
||||
}
|
||||
|
||||
// Then, we find unique mining pool names
|
||||
const poolNames: string[] = [];
|
||||
for (let i = 0; i < poolsDuplicated.length; ++i) {
|
||||
if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
|
||||
poolNames.push(poolsDuplicated[i].name);
|
||||
for (const pool of this.miningPools) {
|
||||
if (!pool.id) {
|
||||
logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);
|
||||
|
||||
// Get existing pools from the db
|
||||
let existingPools;
|
||||
try {
|
||||
if (config.DATABASE.ENABLED === true) {
|
||||
[existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
|
||||
const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
|
||||
if (!poolDB) {
|
||||
// New mining pool
|
||||
const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
|
||||
logger.debug(`Inserting new mining pool ${pool.name}`);
|
||||
await PoolsRepository.$insertNewMiningPool(pool, slug);
|
||||
await this.$deleteUnknownBlocks();
|
||||
} else {
|
||||
existingPools = [];
|
||||
}
|
||||
} catch (e) {
|
||||
logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
|
||||
return;
|
||||
}
|
||||
|
||||
this.miningPools = [];
|
||||
|
||||
// Finally, we generate the final consolidated pools data
|
||||
const finalPoolDataAdd: Pool[] = [];
|
||||
const finalPoolDataUpdate: Pool[] = [];
|
||||
const finalPoolDataRename: Pool[] = [];
|
||||
for (let i = 0; i < poolNames.length; ++i) {
|
||||
let allAddresses: string[] = [];
|
||||
let allRegexes: string[] = [];
|
||||
const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
|
||||
|
||||
for (let y = 0; y < match.length; ++y) {
|
||||
allAddresses = allAddresses.concat(match[y].addresses);
|
||||
allRegexes = allRegexes.concat(match[y].regexes);
|
||||
}
|
||||
|
||||
const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
|
||||
|
||||
let slug: string | undefined;
|
||||
try {
|
||||
slug = poolsJson['slugs'][poolNames[i]];
|
||||
} catch (e) {
|
||||
if (this.slugWarnFlag === false) {
|
||||
logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
|
||||
this.slugWarnFlag = true;
|
||||
if (poolDB.name !== pool.name) {
|
||||
// Pool has been renamed
|
||||
const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
|
||||
logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
|
||||
await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
|
||||
}
|
||||
}
|
||||
|
||||
if (slug === undefined) {
|
||||
// Only keep alphanumerical
|
||||
slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
|
||||
logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
|
||||
}
|
||||
|
||||
const poolObj = {
|
||||
'name': finalPoolName,
|
||||
'link': match[0].link,
|
||||
'regexes': allRegexes,
|
||||
'addresses': allAddresses,
|
||||
'slug': slug
|
||||
};
|
||||
|
||||
const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
|
||||
if (existingPool !== undefined) {
|
||||
// Check if any data was actually updated
|
||||
const equals = (a, b) =>
|
||||
a.length === b.length &&
|
||||
a.every((v, i) => v === b[i]);
|
||||
if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
|
||||
finalPoolDataUpdate.push(poolObj);
|
||||
if (poolDB.link !== pool.link) {
|
||||
// Pool link has changed
|
||||
logger.debug(`Updating link for ${pool.name} mining pool`);
|
||||
await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
|
||||
}
|
||||
} else if (config.DATABASE.ENABLED) {
|
||||
// Double check that if we're not just renaming a pool (same address same regex)
|
||||
const [poolToRename]: any[] = await DB.query(`
|
||||
SELECT * FROM pools
|
||||
WHERE addresses = ? OR regexes = ?`,
|
||||
[JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
|
||||
);
|
||||
if (poolToRename && poolToRename.length > 0) {
|
||||
// We're actually renaming an existing pool
|
||||
finalPoolDataRename.push({
|
||||
'name': poolObj.name,
|
||||
'link': poolObj.link,
|
||||
'regexes': allRegexes,
|
||||
'addresses': allAddresses,
|
||||
'slug': slug
|
||||
});
|
||||
logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
|
||||
} else {
|
||||
logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
|
||||
finalPoolDataAdd.push(poolObj);
|
||||
if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
|
||||
JSON.stringify(pool.regexes) !== poolDB.regexes) {
|
||||
// Pool addresses changed or coinbase tags changed
|
||||
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
|
||||
await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
|
||||
await this.$deleteBlocksForPool(poolDB);
|
||||
}
|
||||
}
|
||||
|
||||
this.miningPools.push({
|
||||
'name': finalPoolName,
|
||||
'link': match[0].link,
|
||||
'regexes': JSON.stringify(allRegexes),
|
||||
'addresses': JSON.stringify(allAddresses),
|
||||
'slug': slug
|
||||
});
|
||||
}
|
||||
|
||||
if (config.DATABASE.ENABLED === false) { // Don't run db operations
|
||||
logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
|
||||
return;
|
||||
}
|
||||
|
||||
if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
|
||||
finalPoolDataRename.length > 0
|
||||
) {
|
||||
logger.debug(`Update pools table now`, logger.tags.mining);
|
||||
|
||||
// Add new mining pools into the database
|
||||
let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
|
||||
for (let i = 0; i < finalPoolDataAdd.length; ++i) {
|
||||
queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
|
||||
'${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
|
||||
${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
|
||||
}
|
||||
queryAdd = queryAdd.slice(0, -1) + ';';
|
||||
|
||||
// Updated existing mining pools in the database
|
||||
const updateQueries: string[] = [];
|
||||
for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
|
||||
updateQueries.push(`
|
||||
UPDATE pools
|
||||
SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
|
||||
regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
|
||||
slug='${finalPoolDataUpdate[i].slug}'
|
||||
WHERE name='${finalPoolDataUpdate[i].name}'
|
||||
;`);
|
||||
}
|
||||
|
||||
// Rename mining pools
|
||||
const renameQueries: string[] = [];
|
||||
for (let i = 0; i < finalPoolDataRename.length; ++i) {
|
||||
renameQueries.push(`
|
||||
UPDATE pools
|
||||
SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
|
||||
slug='${finalPoolDataRename[i].slug}'
|
||||
WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
|
||||
AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
|
||||
;`);
|
||||
}
|
||||
|
||||
try {
|
||||
if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
|
||||
await this.$deleteBlocskToReindex(finalPoolDataUpdate);
|
||||
}
|
||||
|
||||
if (finalPoolDataAdd.length > 0) {
|
||||
await DB.query({ sql: queryAdd, timeout: 120000 });
|
||||
}
|
||||
for (const query of updateQueries) {
|
||||
await DB.query({ sql: query, timeout: 120000 });
|
||||
}
|
||||
for (const query of renameQueries) {
|
||||
await DB.query({ sql: query, timeout: 120000 });
|
||||
}
|
||||
await this.insertUnknownPool();
|
||||
logger.info('Mining pools.json import completed', logger.tags.mining);
|
||||
} catch (e) {
|
||||
logger.err(`Cannot import pools in the database`, logger.tags.mining);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.insertUnknownPool();
|
||||
} catch (e) {
|
||||
logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
|
||||
throw e;
|
||||
}
|
||||
logger.info('Mining pools.json import completed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually add the 'unknown pool'
|
||||
*/
|
||||
private async insertUnknownPool() {
|
||||
public async $insertUnknownPool(): Promise<void> {
|
||||
if (!config.DATABASE.ENABLED) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
|
||||
if (rows.length === 0) {
|
||||
await DB.query({
|
||||
sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
|
||||
VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
|
||||
sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
|
||||
VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
|
||||
`});
|
||||
} else {
|
||||
await DB.query(`UPDATE pools
|
||||
SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
|
||||
SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
|
||||
regexes='[]', addresses='[]',
|
||||
slug='unknown'
|
||||
WHERE name='Unknown'
|
||||
slug='${this.unknownPool.slug}',
|
||||
unique_id=0
|
||||
WHERE slug='${this.unknownPool.slug}'
|
||||
`);
|
||||
}
|
||||
} catch (e) {
|
||||
logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
|
||||
logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete blocks which needs to be reindexed
|
||||
* Delete indexed blocks for an updated mining pool
|
||||
*
|
||||
* @param pool
|
||||
*/
|
||||
private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
|
||||
private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
|
||||
if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
const blockCount = await BlocksRepository.$blockCount(null, null);
|
||||
if (blockCount === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const updatedPool of finalPoolDataUpdate) {
|
||||
const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
|
||||
if (pool.length > 0) {
|
||||
logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
|
||||
await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Ignore early days of Bitcoin as there were not mining pool yet
|
||||
logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
|
||||
// Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years
|
||||
// Ignore early days of Bitcoin as there were no mining pool yet
|
||||
const [oldestPoolBlock]: any[] = await DB.query(`
|
||||
SELECT height
|
||||
FROM blocks
|
||||
WHERE pool_id = ?
|
||||
ORDER BY height
|
||||
LIMIT 1`,
|
||||
[pool.id]
|
||||
);
|
||||
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
|
||||
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
|
||||
await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
|
||||
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
|
||||
await DB.query(`
|
||||
DELETE FROM blocks
|
||||
WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
|
||||
[unknownPool[0].id]
|
||||
);
|
||||
logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
|
||||
await DB.query(`
|
||||
DELETE FROM blocks
|
||||
WHERE pool_id = ?`,
|
||||
[pool.id]
|
||||
);
|
||||
}
|
||||
|
||||
logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
|
||||
await DB.query(`DELETE FROM hashrates`);
|
||||
private async $deleteUnknownBlocks(): Promise<void> {
|
||||
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
|
||||
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
|
||||
await DB.query(`
|
||||
DELETE FROM blocks
|
||||
WHERE pool_id = ? AND height >= 130635`,
|
||||
[unknownPool[0].id]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ class TransactionUtils {
|
||||
vout: tx.vout
|
||||
.map((vout) => ({
|
||||
scriptpubkey_address: vout.scriptpubkey_address,
|
||||
scriptpubkey_asm: vout.scriptpubkey_asm,
|
||||
value: vout.value
|
||||
}))
|
||||
.filter((vout) => vout.value)
|
||||
|
||||
Reference in New Issue
Block a user