Merge branch 'master' into nymkappa/bugfix/wipe-cache-reindexing
This commit is contained in:
softsimon
GitHub
@@ -119,7 +119,8 @@ class Audit {
|
||||
}
|
||||
|
||||
const numCensored = Object.keys(isCensored).length;
|
||||
const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
|
||||
const numMatches = matches.length - 1; // adjust for coinbase tx
|
||||
const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
|
||||
|
||||
return {
|
||||
censored: Object.keys(isCensored),
|
||||
|
||||
@@ -172,4 +172,35 @@ export namespace IBitcoinApi {
|
||||
}
|
||||
}
|
||||
|
||||
export interface BlockStats {
|
||||
"avgfee": number;
|
||||
"avgfeerate": number;
|
||||
"avgtxsize": number;
|
||||
"blockhash": string;
|
||||
"feerate_percentiles": [number, number, number, number, number];
|
||||
"height": number;
|
||||
"ins": number;
|
||||
"maxfee": number;
|
||||
"maxfeerate": number;
|
||||
"maxtxsize": number;
|
||||
"medianfee": number;
|
||||
"mediantime": number;
|
||||
"mediantxsize": number;
|
||||
"minfee": number;
|
||||
"minfeerate": number;
|
||||
"mintxsize": number;
|
||||
"outs": number;
|
||||
"subsidy": number;
|
||||
"swtotal_size": number;
|
||||
"swtotal_weight": number;
|
||||
"swtxs": number;
|
||||
"time": number;
|
||||
"total_out": number;
|
||||
"total_size": number;
|
||||
"total_weight": number;
|
||||
"totalfee": number;
|
||||
"txs": number;
|
||||
"utxo_increase": number;
|
||||
"utxo_size_inc": number;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
||||
size: block.size,
|
||||
weight: block.weight,
|
||||
previousblockhash: block.previousblockhash,
|
||||
medianTime: block.mediantime,
|
||||
mediantime: block.mediantime,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -217,7 +217,15 @@ class BitcoinRoutes {
|
||||
res.json(cpfpInfo);
|
||||
return;
|
||||
} else {
|
||||
const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
|
||||
let cpfpInfo;
|
||||
if (config.DATABASE.ENABLED) {
|
||||
cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
|
||||
} else {
|
||||
res.json({
|
||||
ancestors: []
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (cpfpInfo) {
|
||||
res.json(cpfpInfo);
|
||||
return;
|
||||
|
||||
@@ -88,7 +88,7 @@ export namespace IEsploraApi {
|
||||
size: number;
|
||||
weight: number;
|
||||
previousblockhash: string;
|
||||
medianTime?: number;
|
||||
mediantime: number;
|
||||
}
|
||||
|
||||
export interface Address {
|
||||
|
||||
@@ -2,7 +2,7 @@ import config from '../config';
|
||||
import bitcoinApi from './bitcoin/bitcoin-api-factory';
|
||||
import logger from '../logger';
|
||||
import memPool from './mempool';
|
||||
import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
|
||||
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
|
||||
import { Common } from './common';
|
||||
import diskCache from './disk-cache';
|
||||
import transactionUtils from './transaction-utils';
|
||||
@@ -13,7 +13,6 @@ import poolsRepository from '../repositories/PoolsRepository';
|
||||
import blocksRepository from '../repositories/BlocksRepository';
|
||||
import loadingIndicators from './loading-indicators';
|
||||
import BitcoinApi from './bitcoin/bitcoin-api';
|
||||
import { prepareBlock } from '../utils/blocks-utils';
|
||||
import BlocksRepository from '../repositories/BlocksRepository';
|
||||
import HashratesRepository from '../repositories/HashratesRepository';
|
||||
import indexer from '../indexer';
|
||||
@@ -143,7 +142,7 @@ class Blocks {
|
||||
* @param block
|
||||
* @returns BlockSummary
|
||||
*/
|
||||
private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
|
||||
public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
|
||||
const stripped = block.tx.map((tx) => {
|
||||
return {
|
||||
txid: tx.txid,
|
||||
@@ -166,80 +165,81 @@ class Blocks {
|
||||
* @returns BlockExtended
|
||||
*/
|
||||
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
|
||||
const blk: BlockExtended = Object.assign({ extras: {} }, block);
|
||||
blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
|
||||
blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
|
||||
blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
|
||||
blk.extras.usd = priceUpdater.latestPrices.USD;
|
||||
blk.extras.medianTimestamp = block.medianTime;
|
||||
blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
|
||||
const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
|
||||
|
||||
const blk: Partial<BlockExtended> = Object.assign({}, block);
|
||||
const extras: Partial<BlockExtension> = {};
|
||||
|
||||
extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
|
||||
extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
|
||||
extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
|
||||
|
||||
if (block.height === 0) {
|
||||
blk.extras.medianFee = 0; // 50th percentiles
|
||||
blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
|
||||
blk.extras.totalFees = 0;
|
||||
blk.extras.avgFee = 0;
|
||||
blk.extras.avgFeeRate = 0;
|
||||
blk.extras.utxoSetChange = 0;
|
||||
blk.extras.avgTxSize = 0;
|
||||
blk.extras.totalInputs = 0;
|
||||
blk.extras.totalOutputs = 1;
|
||||
blk.extras.totalOutputAmt = 0;
|
||||
blk.extras.segwitTotalTxs = 0;
|
||||
blk.extras.segwitTotalSize = 0;
|
||||
blk.extras.segwitTotalWeight = 0;
|
||||
extras.medianFee = 0; // 50th percentiles
|
||||
extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
|
||||
extras.totalFees = 0;
|
||||
extras.avgFee = 0;
|
||||
extras.avgFeeRate = 0;
|
||||
extras.utxoSetChange = 0;
|
||||
extras.avgTxSize = 0;
|
||||
extras.totalInputs = 0;
|
||||
extras.totalOutputs = 1;
|
||||
extras.totalOutputAmt = 0;
|
||||
extras.segwitTotalTxs = 0;
|
||||
extras.segwitTotalSize = 0;
|
||||
extras.segwitTotalWeight = 0;
|
||||
} else {
|
||||
const stats = await bitcoinClient.getBlockStats(block.id);
|
||||
blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
|
||||
blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
|
||||
blk.extras.totalFees = stats.totalfee;
|
||||
blk.extras.avgFee = stats.avgfee;
|
||||
blk.extras.avgFeeRate = stats.avgfeerate;
|
||||
blk.extras.utxoSetChange = stats.utxo_increase;
|
||||
blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
|
||||
blk.extras.totalInputs = stats.ins;
|
||||
blk.extras.totalOutputs = stats.outs;
|
||||
blk.extras.totalOutputAmt = stats.total_out;
|
||||
blk.extras.segwitTotalTxs = stats.swtxs;
|
||||
blk.extras.segwitTotalSize = stats.swtotal_size;
|
||||
blk.extras.segwitTotalWeight = stats.swtotal_weight;
|
||||
const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
|
||||
extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
|
||||
extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
|
||||
extras.totalFees = stats.totalfee;
|
||||
extras.avgFee = stats.avgfee;
|
||||
extras.avgFeeRate = stats.avgfeerate;
|
||||
extras.utxoSetChange = stats.utxo_increase;
|
||||
extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
|
||||
extras.totalInputs = stats.ins;
|
||||
extras.totalOutputs = stats.outs;
|
||||
extras.totalOutputAmt = stats.total_out;
|
||||
extras.segwitTotalTxs = stats.swtxs;
|
||||
extras.segwitTotalSize = stats.swtotal_size;
|
||||
extras.segwitTotalWeight = stats.swtotal_weight;
|
||||
}
|
||||
|
||||
if (Common.blocksSummariesIndexingEnabled()) {
|
||||
blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
|
||||
if (blk.extras.feePercentiles !== null) {
|
||||
blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
|
||||
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
|
||||
if (extras.feePercentiles !== null) {
|
||||
extras.medianFeeAmt = extras.feePercentiles[3];
|
||||
}
|
||||
}
|
||||
|
||||
blk.extras.virtualSize = block.weight / 4.0;
|
||||
if (blk.extras.coinbaseTx.vout.length > 0) {
|
||||
blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
|
||||
blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
|
||||
blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null;
|
||||
extras.virtualSize = block.weight / 4.0;
|
||||
if (coinbaseTx?.vout.length > 0) {
|
||||
extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
|
||||
extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
|
||||
extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
|
||||
} else {
|
||||
blk.extras.coinbaseAddress = null;
|
||||
blk.extras.coinbaseSignature = null;
|
||||
blk.extras.coinbaseSignatureAscii = null;
|
||||
extras.coinbaseAddress = null;
|
||||
extras.coinbaseSignature = null;
|
||||
extras.coinbaseSignatureAscii = null;
|
||||
}
|
||||
|
||||
const header = await bitcoinClient.getBlockHeader(block.id, false);
|
||||
blk.extras.header = header;
|
||||
extras.header = header;
|
||||
|
||||
const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
|
||||
if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
|
||||
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
|
||||
blk.extras.utxoSetSize = txoutset.txouts,
|
||||
blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
|
||||
extras.utxoSetSize = txoutset.txouts,
|
||||
extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
|
||||
} else {
|
||||
blk.extras.utxoSetSize = null;
|
||||
blk.extras.totalInputAmt = null;
|
||||
extras.utxoSetSize = null;
|
||||
extras.totalInputAmt = null;
|
||||
}
|
||||
|
||||
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
|
||||
let pool: PoolTag;
|
||||
if (blk.extras?.coinbaseTx !== undefined) {
|
||||
pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
|
||||
if (coinbaseTx !== undefined) {
|
||||
pool = await this.$findBlockMiner(coinbaseTx);
|
||||
} else {
|
||||
if (config.DATABASE.ENABLED === true) {
|
||||
pool = await poolsRepository.$getUnknownPool();
|
||||
@@ -252,22 +252,24 @@ class Blocks {
|
||||
logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
|
||||
`Check your "pools" table entries`);
|
||||
} else {
|
||||
blk.extras.pool = {
|
||||
id: pool.id,
|
||||
extras.pool = {
|
||||
id: pool.uniqueId,
|
||||
name: pool.name,
|
||||
slug: pool.slug,
|
||||
};
|
||||
}
|
||||
|
||||
extras.matchRate = null;
|
||||
if (config.MEMPOOL.AUDIT) {
|
||||
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
|
||||
if (auditScore != null) {
|
||||
blk.extras.matchRate = auditScore.matchRate;
|
||||
extras.matchRate = auditScore.matchRate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return blk;
|
||||
blk.extras = <BlockExtension>extras;
|
||||
return <BlockExtended>blk;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -293,15 +295,18 @@ class Blocks {
|
||||
} else {
|
||||
pools = poolsParser.miningPools;
|
||||
}
|
||||
|
||||
for (let i = 0; i < pools.length; ++i) {
|
||||
if (address !== undefined) {
|
||||
const addresses: string[] = JSON.parse(pools[i].addresses);
|
||||
const addresses: string[] = typeof pools[i].addresses === 'string' ?
|
||||
JSON.parse(pools[i].addresses) : pools[i].addresses;
|
||||
if (addresses.indexOf(address) !== -1) {
|
||||
return pools[i];
|
||||
}
|
||||
}
|
||||
|
||||
const regexes: string[] = JSON.parse(pools[i].regexes);
|
||||
const regexes: string[] = typeof pools[i].regexes === 'string' ?
|
||||
JSON.parse(pools[i].regexes) : pools[i].regexes;
|
||||
for (let y = 0; y < regexes.length; ++y) {
|
||||
const regex = new RegExp(regexes[y], 'i');
|
||||
const match = asciiScriptSig.match(regex);
|
||||
@@ -479,7 +484,7 @@ class Blocks {
|
||||
loadingIndicators.setProgress('block-indexing', progress, false);
|
||||
}
|
||||
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
|
||||
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
|
||||
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
|
||||
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
|
||||
const blockExtended = await this.$getBlockExtended(block, transactions);
|
||||
|
||||
@@ -527,13 +532,13 @@ class Blocks {
|
||||
if (blockchainInfo.blocks === blockchainInfo.headers) {
|
||||
const heightDiff = blockHeightTip % 2016;
|
||||
const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
|
||||
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
|
||||
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
|
||||
this.lastDifficultyAdjustmentTime = block.timestamp;
|
||||
this.currentDifficulty = block.difficulty;
|
||||
|
||||
if (blockHeightTip >= 2016) {
|
||||
const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
|
||||
const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
|
||||
const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
|
||||
this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
|
||||
logger.debug(`Initial difficulty adjustment data set.`);
|
||||
}
|
||||
@@ -565,18 +570,18 @@ class Blocks {
|
||||
if (Common.indexingEnabled()) {
|
||||
if (!fastForwarded) {
|
||||
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
|
||||
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
|
||||
logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
|
||||
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
|
||||
logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
|
||||
// We assume there won't be a reorg with more than 10 block depth
|
||||
await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
|
||||
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
|
||||
await HashratesRepository.$deleteLastEntries();
|
||||
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
|
||||
await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
|
||||
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
|
||||
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
|
||||
for (let i = 10; i >= 0; --i) {
|
||||
const newBlock = await this.$indexBlock(lastBlock['height'] - i);
|
||||
const newBlock = await this.$indexBlock(lastBlock.height - i);
|
||||
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
|
||||
if (config.MEMPOOL.CPFP_INDEXING) {
|
||||
await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
|
||||
await this.$indexCPFP(newBlock.id, lastBlock.height - i);
|
||||
}
|
||||
}
|
||||
await mining.$indexDifficultyAdjustments();
|
||||
@@ -652,12 +657,12 @@ class Blocks {
|
||||
if (Common.indexingEnabled()) {
|
||||
const dbBlock = await blocksRepository.$getBlockByHeight(height);
|
||||
if (dbBlock !== null) {
|
||||
return prepareBlock(dbBlock);
|
||||
return dbBlock;
|
||||
}
|
||||
}
|
||||
|
||||
const blockHash = await bitcoinApi.$getBlockHash(height);
|
||||
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
|
||||
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
|
||||
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
|
||||
const blockExtended = await this.$getBlockExtended(block, transactions);
|
||||
|
||||
@@ -665,11 +670,11 @@ class Blocks {
|
||||
await blocksRepository.$saveBlockInDatabase(blockExtended);
|
||||
}
|
||||
|
||||
return prepareBlock(blockExtended);
|
||||
return blockExtended;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a block by hash if it's missing from the database. Returns the block after indexing
|
||||
* Get one block by its hash
|
||||
*/
|
||||
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
|
||||
// Check the memory cache
|
||||
@@ -678,31 +683,14 @@ class Blocks {
|
||||
return blockByHash;
|
||||
}
|
||||
|
||||
// Block has already been indexed
|
||||
if (Common.indexingEnabled()) {
|
||||
const dbBlock = await blocksRepository.$getBlockByHash(hash);
|
||||
if (dbBlock != null) {
|
||||
return prepareBlock(dbBlock);
|
||||
}
|
||||
}
|
||||
|
||||
// Not Bitcoin network, return the block as it
|
||||
// Not Bitcoin network, return the block as it from the bitcoin backend
|
||||
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
|
||||
return await bitcoinApi.$getBlock(hash);
|
||||
}
|
||||
|
||||
let block = await bitcoinClient.getBlock(hash);
|
||||
block = prepareBlock(block);
|
||||
|
||||
// Bitcoin network, add our custom data on top
|
||||
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
|
||||
const blockExtended = await this.$getBlockExtended(block, transactions);
|
||||
if (Common.indexingEnabled()) {
|
||||
delete(blockExtended['coinbaseTx']);
|
||||
await blocksRepository.$saveBlockInDatabase(blockExtended);
|
||||
}
|
||||
|
||||
return blockExtended;
|
||||
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
|
||||
return await this.$indexBlock(block.height);
|
||||
}
|
||||
|
||||
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
|
||||
@@ -736,6 +724,18 @@ class Blocks {
|
||||
return summary.transactions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get 15 blocks
|
||||
*
|
||||
* Internally this function uses two methods to get the blocks, and
|
||||
* the method is automatically selected:
|
||||
* - Using previous block hash links
|
||||
* - Using block height
|
||||
*
|
||||
* @param fromHeight
|
||||
* @param limit
|
||||
* @returns
|
||||
*/
|
||||
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
|
||||
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
|
||||
if (currentHeight > this.currentBlockHeight) {
|
||||
@@ -748,27 +748,15 @@ class Blocks {
|
||||
return returnBlocks;
|
||||
}
|
||||
|
||||
// Check if block height exist in local cache to skip the hash lookup
|
||||
const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
|
||||
let startFromHash: string | null = null;
|
||||
if (blockByHeight) {
|
||||
startFromHash = blockByHeight.id;
|
||||
} else if (!Common.indexingEnabled()) {
|
||||
startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
|
||||
}
|
||||
|
||||
let nextHash = startFromHash;
|
||||
for (let i = 0; i < limit && currentHeight >= 0; i++) {
|
||||
let block = this.getBlocks().find((b) => b.height === currentHeight);
|
||||
if (block) {
|
||||
// Using the memory cache (find by height)
|
||||
returnBlocks.push(block);
|
||||
} else if (Common.indexingEnabled()) {
|
||||
} else {
|
||||
// Using indexing (find by height, index on the fly, save in database)
|
||||
block = await this.$indexBlock(currentHeight);
|
||||
returnBlocks.push(block);
|
||||
} else if (nextHash != null) {
|
||||
block = await this.$indexBlock(currentHeight);
|
||||
nextHash = block.previousblockhash;
|
||||
returnBlocks.push(block);
|
||||
}
|
||||
currentHeight--;
|
||||
}
|
||||
@@ -790,7 +778,7 @@ class Blocks {
|
||||
const blocks: any[] = [];
|
||||
|
||||
while (fromHeight <= toHeight) {
|
||||
let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
|
||||
let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
|
||||
if (!block) {
|
||||
await this.$indexBlock(fromHeight);
|
||||
block = await blocksRepository.$getBlockByHeight(fromHeight);
|
||||
@@ -803,11 +791,11 @@ class Blocks {
|
||||
const cleanBlock: any = {
|
||||
height: block.height ?? null,
|
||||
hash: block.id ?? null,
|
||||
timestamp: block.blockTimestamp ?? null,
|
||||
median_timestamp: block.medianTime ?? null,
|
||||
timestamp: block.timestamp ?? null,
|
||||
median_timestamp: block.mediantime ?? null,
|
||||
previous_block_hash: block.previousblockhash ?? null,
|
||||
difficulty: block.difficulty ?? null,
|
||||
header: block.header ?? null,
|
||||
header: block.extras.header ?? null,
|
||||
version: block.version ?? null,
|
||||
bits: block.bits ?? null,
|
||||
nonce: block.nonce ?? null,
|
||||
@@ -815,29 +803,30 @@ class Blocks {
|
||||
weight: block.weight ?? null,
|
||||
tx_count: block.tx_count ?? null,
|
||||
merkle_root: block.merkle_root ?? null,
|
||||
reward: block.reward ?? null,
|
||||
total_fee_amt: block.fees ?? null,
|
||||
avg_fee_amt: block.avg_fee ?? null,
|
||||
median_fee_amt: block.median_fee_amt ?? null,
|
||||
fee_amt_percentiles: block.fee_percentiles ?? null,
|
||||
avg_fee_rate: block.avg_fee_rate ?? null,
|
||||
median_fee_rate: block.median_fee ?? null,
|
||||
fee_rate_percentiles: block.fee_span ?? null,
|
||||
total_inputs: block.total_inputs ?? null,
|
||||
total_input_amt: block.total_input_amt ?? null,
|
||||
total_outputs: block.total_outputs ?? null,
|
||||
total_output_amt: block.total_output_amt ?? null,
|
||||
segwit_total_txs: block.segwit_total_txs ?? null,
|
||||
segwit_total_size: block.segwit_total_size ?? null,
|
||||
segwit_total_weight: block.segwit_total_weight ?? null,
|
||||
avg_tx_size: block.avg_tx_size ?? null,
|
||||
utxoset_change: block.utxoset_change ?? null,
|
||||
utxoset_size: block.utxoset_size ?? null,
|
||||
coinbase_raw: block.coinbase_raw ?? null,
|
||||
coinbase_address: block.coinbase_address ?? null,
|
||||
coinbase_signature: block.coinbase_signature ?? null,
|
||||
coinbase_signature_ascii: block.coinbase_signature_ascii ?? null,
|
||||
pool_slug: block.pool_slug ?? null,
|
||||
reward: block.extras.reward ?? null,
|
||||
total_fee_amt: block.extras.totalFees ?? null,
|
||||
avg_fee_amt: block.extras.avgFee ?? null,
|
||||
median_fee_amt: block.extras.medianFeeAmt ?? null,
|
||||
fee_amt_percentiles: block.extras.feePercentiles ?? null,
|
||||
avg_fee_rate: block.extras.avgFeeRate ?? null,
|
||||
median_fee_rate: block.extras.medianFee ?? null,
|
||||
fee_rate_percentiles: block.extras.feeRange ?? null,
|
||||
total_inputs: block.extras.totalInputs ?? null,
|
||||
total_input_amt: block.extras.totalInputAmt ?? null,
|
||||
total_outputs: block.extras.totalOutputs ?? null,
|
||||
total_output_amt: block.extras.totalOutputAmt ?? null,
|
||||
segwit_total_txs: block.extras.segwitTotalTxs ?? null,
|
||||
segwit_total_size: block.extras.segwitTotalSize ?? null,
|
||||
segwit_total_weight: block.extras.segwitTotalWeight ?? null,
|
||||
avg_tx_size: block.extras.avgTxSize ?? null,
|
||||
utxoset_change: block.extras.utxoSetChange ?? null,
|
||||
utxoset_size: block.extras.utxoSetSize ?? null,
|
||||
coinbase_raw: block.extras.coinbaseRaw ?? null,
|
||||
coinbase_address: block.extras.coinbaseAddress ?? null,
|
||||
coinbase_signature: block.extras.coinbaseSignature ?? null,
|
||||
coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
|
||||
pool_slug: block.extras.pool.slug ?? null,
|
||||
pool_id: block.extras.pool.id ?? null,
|
||||
};
|
||||
|
||||
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import logger from "../logger";
|
||||
import bitcoinClient from "./bitcoin/bitcoin-client";
|
||||
import logger from '../logger';
|
||||
import bitcoinClient from './bitcoin/bitcoin-client';
|
||||
|
||||
export interface ChainTip {
|
||||
height: number;
|
||||
@@ -43,7 +43,11 @@ class ChainTips {
|
||||
}
|
||||
}
|
||||
|
||||
public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {
|
||||
public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
|
||||
if (height === undefined) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const orphans: OrphanedBlock[] = [];
|
||||
for (const block of this.orphanedBlocks) {
|
||||
if (block.height === height) {
|
||||
|
||||
@@ -237,14 +237,21 @@ export class Common {
|
||||
].join('x');
|
||||
}
|
||||
|
||||
static utcDateToMysql(date?: number): string {
|
||||
static utcDateToMysql(date?: number | null): string | null {
|
||||
if (date === null) {
|
||||
return null;
|
||||
}
|
||||
const d = new Date((date || 0) * 1000);
|
||||
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
|
||||
}
|
||||
|
||||
static findSocketNetwork(addr: string): {network: string | null, url: string} {
|
||||
let network: string | null = null;
|
||||
let url = addr.split('://')[1];
|
||||
let url: string = addr;
|
||||
|
||||
if (config.LIGHTNING.BACKEND === 'cln') {
|
||||
url = addr.split('://')[1];
|
||||
}
|
||||
|
||||
if (!url) {
|
||||
return {
|
||||
@@ -261,7 +268,7 @@ export class Common {
|
||||
}
|
||||
} else if (addr.indexOf('i2p') !== -1) {
|
||||
network = 'i2p';
|
||||
} else if (addr.indexOf('ipv4') !== -1) {
|
||||
} else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
|
||||
const ipv = isIP(url.split(':')[0]);
|
||||
if (ipv === 4) {
|
||||
network = 'ipv4';
|
||||
@@ -271,7 +278,7 @@ export class Common {
|
||||
url: addr,
|
||||
};
|
||||
}
|
||||
} else if (addr.indexOf('ipv6') !== -1) {
|
||||
} else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
|
||||
url = url.split('[')[1].split(']')[0];
|
||||
const ipv = isIP(url);
|
||||
if (ipv === 6) {
|
||||
|
||||
@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
|
||||
import { RowDataPacket } from 'mysql2';
|
||||
|
||||
class DatabaseMigration {
|
||||
private static currentVersion = 56;
|
||||
private static currentVersion = 57;
|
||||
private queryTimeout = 3600_000;
|
||||
private statisticsAddedIndexed = false;
|
||||
private uniqueLogs: string[] = [];
|
||||
@@ -500,6 +500,11 @@ class DatabaseMigration {
|
||||
this.uniqueLog(logger.notice, '`pools` table has been truncated`');
|
||||
await this.updateToSchemaVersion(56);
|
||||
}
|
||||
|
||||
if (databaseSchemaVersion < 57 && isBitcoin === true) {
|
||||
await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
|
||||
await this.updateToSchemaVersion(57);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1012,26 +1017,16 @@ class DatabaseMigration {
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
|
||||
}
|
||||
|
||||
public async $truncateIndexedData(tables: string[]) {
|
||||
const allowedTables = ['blocks', 'hashrates', 'prices'];
|
||||
public async $blocksReindexingTruncate(): Promise<void> {
|
||||
logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
|
||||
await Common.sleep$(5000);
|
||||
|
||||
try {
|
||||
for (const table of tables) {
|
||||
if (!allowedTables.includes(table)) {
|
||||
logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.$executeQuery(`TRUNCATE ${table}`, true);
|
||||
if (table === 'hashrates') {
|
||||
await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
|
||||
}
|
||||
logger.notice(`Table ${table} has been truncated`);
|
||||
}
|
||||
} catch (e) {
|
||||
logger.warn(`Unable to erase indexed data`);
|
||||
}
|
||||
}
|
||||
await this.$executeQuery(`TRUNCATE blocks`);
|
||||
await this.$executeQuery(`TRUNCATE hashrates`);
|
||||
await this.$executeQuery('DELETE FROM `pools`');
|
||||
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
|
||||
await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
|
||||
}
|
||||
|
||||
private async $convertCompactCpfpTables(): Promise<void> {
|
||||
try {
|
||||
|
||||
@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
|
||||
import { Common } from './common';
|
||||
|
||||
class DiskCache {
|
||||
private cacheSchemaVersion = 2;
|
||||
private cacheSchemaVersion = 3;
|
||||
|
||||
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
|
||||
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
|
||||
|
||||
@@ -559,6 +559,17 @@ class ChannelsApi {
|
||||
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
|
||||
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
|
||||
|
||||
// https://github.com/mempool/mempool/issues/3006
|
||||
if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
|
||||
channel.last_update = null;
|
||||
}
|
||||
if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
|
||||
policy1.last_update = null;
|
||||
}
|
||||
if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
|
||||
policy2.last_update = null;
|
||||
}
|
||||
|
||||
const query = `INSERT INTO channels
|
||||
(
|
||||
id,
|
||||
|
||||
@@ -228,7 +228,7 @@ class NodesApi {
|
||||
nodes.capacity
|
||||
FROM nodes
|
||||
ORDER BY capacity DESC
|
||||
LIMIT 100
|
||||
LIMIT 6
|
||||
`;
|
||||
|
||||
[rows] = await DB.query(query);
|
||||
@@ -269,14 +269,26 @@ class NodesApi {
|
||||
let query: string;
|
||||
if (full === false) {
|
||||
query = `
|
||||
SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
|
||||
nodes.channels
|
||||
SELECT
|
||||
nodes.public_key as publicKey,
|
||||
IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
|
||||
nodes.channels,
|
||||
geo_names_city.names as city, geo_names_country.names as country,
|
||||
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
|
||||
FROM nodes
|
||||
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
|
||||
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
|
||||
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
|
||||
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
|
||||
ORDER BY channels DESC
|
||||
LIMIT 100;
|
||||
LIMIT 6;
|
||||
`;
|
||||
|
||||
[rows] = await DB.query(query);
|
||||
for (let i = 0; i < rows.length; ++i) {
|
||||
rows[i].country = JSON.parse(rows[i].country);
|
||||
rows[i].city = JSON.parse(rows[i].city);
|
||||
}
|
||||
} else {
|
||||
query = `
|
||||
SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
|
||||
@@ -630,6 +642,11 @@ class NodesApi {
|
||||
*/
|
||||
public async $saveNode(node: ILightningApi.Node): Promise<void> {
|
||||
try {
|
||||
// https://github.com/mempool/mempool/issues/3006
|
||||
if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
|
||||
node.last_update = null;
|
||||
}
|
||||
|
||||
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
|
||||
const query = `INSERT INTO nodes(
|
||||
public_key,
|
||||
|
||||
@@ -21,7 +21,7 @@ export namespace ILightningApi {
|
||||
export interface Channel {
|
||||
channel_id: string;
|
||||
chan_point: string;
|
||||
last_update: number;
|
||||
last_update: number | null;
|
||||
node1_pub: string;
|
||||
node2_pub: string;
|
||||
capacity: string;
|
||||
@@ -36,11 +36,11 @@ export namespace ILightningApi {
|
||||
fee_rate_milli_msat: string;
|
||||
disabled: boolean;
|
||||
max_htlc_msat: string;
|
||||
last_update: number;
|
||||
last_update: number | null;
|
||||
}
|
||||
|
||||
export interface Node {
|
||||
last_update: number;
|
||||
last_update: number | null;
|
||||
pub_key: string;
|
||||
alias: string;
|
||||
addresses: {
|
||||
|
||||
@@ -11,6 +11,8 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
|
||||
import config from '../../config';
|
||||
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
|
||||
import PricesRepository from '../../repositories/PricesRepository';
|
||||
import bitcoinApiFactory from '../bitcoin/bitcoin-api-factory';
|
||||
import { IEsploraApi } from '../bitcoin/esplora-api.interface';
|
||||
|
||||
class Mining {
|
||||
blocksPriceIndexingRunning = false;
|
||||
@@ -189,8 +191,8 @@ class Mining {
|
||||
try {
|
||||
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
|
||||
|
||||
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
|
||||
const genesisTimestamp = genesisBlock.time * 1000;
|
||||
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
|
||||
const genesisTimestamp = genesisBlock.timestamp * 1000;
|
||||
|
||||
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
|
||||
const hashrates: any[] = [];
|
||||
@@ -292,8 +294,8 @@ class Mining {
|
||||
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
|
||||
|
||||
try {
|
||||
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
|
||||
const genesisTimestamp = genesisBlock.time * 1000;
|
||||
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
|
||||
const genesisTimestamp = genesisBlock.timestamp * 1000;
|
||||
const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
|
||||
const lastMidnight = this.getDateMidnight(new Date());
|
||||
let toTimestamp = Math.round(lastMidnight.getTime());
|
||||
@@ -394,13 +396,13 @@ class Mining {
|
||||
}
|
||||
|
||||
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
|
||||
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
|
||||
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
|
||||
let currentDifficulty = genesisBlock.difficulty;
|
||||
let totalIndexed = 0;
|
||||
|
||||
if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
|
||||
await DifficultyAdjustmentsRepository.$saveAdjustments({
|
||||
time: genesisBlock.time,
|
||||
time: genesisBlock.timestamp,
|
||||
height: 0,
|
||||
difficulty: currentDifficulty,
|
||||
adjustment: 0.0,
|
||||
|
||||
@@ -8,6 +8,7 @@ import diskCache from './disk-cache';
|
||||
class PoolsParser {
|
||||
miningPools: any[] = [];
|
||||
unknownPool: any = {
|
||||
'id': 0,
|
||||
'name': 'Unknown',
|
||||
'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
|
||||
'regexes': '[]',
|
||||
@@ -27,6 +28,7 @@ class PoolsParser {
|
||||
public setMiningPools(pools): void {
|
||||
for (const pool of pools) {
|
||||
pool.regexes = pool.tags;
|
||||
pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
|
||||
delete(pool.tags);
|
||||
}
|
||||
this.miningPools = pools;
|
||||
@@ -74,7 +76,7 @@ class PoolsParser {
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('Mining pools.json import completed');
|
||||
logger.info('Mining pools-v2.json import completed');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -116,7 +118,7 @@ class PoolsParser {
|
||||
return;
|
||||
}
|
||||
|
||||
// Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years
|
||||
// Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
|
||||
// Ignore early days of Bitcoin as there were no mining pool yet
|
||||
const [oldestPoolBlock]: any[] = await DB.query(`
|
||||
SELECT height
|
||||
|
||||
Reference in New Issue
Block a user