machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enforce BlockExtended use for block indexing - Unify /api/v1/block(s) API(s) response format

Browse Source
This commit is contained in:
nymkappa
2023-02-27 18:00:00 +09:00
parent 5792dee553
commit 0aff276a5c
12 changed files with 352 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
backend/src/repositories/BlocksRepository.ts
View File

@@ -1,8 +1,7 @@
import { BlockExtended, BlockPrice } from '../mempool.interfaces';
import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
import DB from '../database';
import logger from '../logger';
import { Common } from '../api/common';
import { prepareBlock } from '../utils/blocks-utils';
import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2';
@@ -10,6 +9,50 @@ import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
import bitcoinClient from '../api/bitcoin/bitcoin-client';
import config from '../config';
import chainTips from '../api/chain-tips';
import blocks from '../api/blocks';
const BLOCK_DB_FIELDS = `
blocks.hash AS id,
blocks.height,
blocks.version,
UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
blocks.bits,
blocks.nonce,
blocks.difficulty,
blocks.merkle_root,
blocks.tx_count,
blocks.size,
blocks.weight,
blocks.previous_block_hash AS previousblockhash,
UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
blocks.fees AS totalFees,
blocks.median_fee AS medianFee,
blocks.fee_span AS feeRange,
blocks.reward,
pools.unique_id AS poolId,
pools.name AS poolName,
pools.slug AS poolSlug,
blocks.avg_fee AS avgFee,
blocks.avg_fee_rate AS avgFeeRate,
blocks.coinbase_raw AS coinbaseRaw,
blocks.coinbase_address AS coinbaseAddress,
blocks.coinbase_signature AS coinbaseSignature,
blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
blocks.avg_tx_size AS avgTxSize,
blocks.total_inputs AS totalInputs,
blocks.total_outputs AS totalOutputs,
blocks.total_output_amt AS totalOutputAmt,
blocks.median_fee_amt AS medianFeeAmt,
blocks.fee_percentiles AS feePercentiles,
blocks.segwit_total_txs AS segwitTotalTxs,
blocks.segwit_total_size AS segwitTotalSize,
blocks.segwit_total_weight AS segwitTotalWeight,
blocks.header,
blocks.utxoset_change AS utxoSetChange,
blocks.utxoset_size AS utxoSetSize,
blocks.total_input_amt AS totalInputAmts
`;
class BlocksRepository {
/**
@@ -44,6 +87,11 @@ class BlocksRepository {
?, ?
)`;
const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
if (!poolDbId) {
throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
}
const params: any[] = [
block.height,
block.id,
@@ -53,7 +101,7 @@ class BlocksRepository {
block.tx_count,
block.extras.coinbaseRaw,
block.difficulty,
block.extras.pool?.id, // Should always be set to something
poolDbId.id,
block.extras.totalFees,
JSON.stringify(block.extras.feeRange),
block.extras.medianFee,
@@ -65,7 +113,7 @@ class BlocksRepository {
block.previousblockhash,
block.extras.avgFee,
block.extras.avgFeeRate,
block.extras.medianTimestamp,
block.mediantime,
block.extras.header,
block.extras.coinbaseAddress,
truncatedCoinbaseSignature,
@@ -87,9 +135,9 @@ class BlocksRepository {
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
} else {
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@@ -307,34 +355,17 @@ class BlocksRepository {
/**
* Get blocks mined by a specific mining pool
*/
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error('This mining pool does not exist ' + escape(slug));
}
const params: any[] = [];
let query = ` SELECT
blocks.height,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
let query = `
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE pool_id = ?`;
params.push(pool.id);
@@ -347,11 +378,11 @@ class BlocksRepository {
LIMIT 10`;
try {
const [rows] = await DB.query(query, params);
const [rows]: any[] = await DB.query(query, params);
const blocks: BlockExtended[] = [];
for (const block of <object[]>rows) {
blocks.push(prepareBlock(block));
for (const block of rows) {
blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
}
return blocks;
@@ -364,32 +395,21 @@ class BlocksRepository {
/**
* Get one block by height
*/
public async $getBlockByHeight(height: number): Promise<object | null> {
public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
try {
const [rows]: any[] = await DB.query(`SELECT
blocks.*,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime,
pools.id as pool_id,
pools.name as pool_name,
pools.link as pool_link,
pools.slug as pool_slug,
pools.addresses as pool_addresses,
pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
const [rows]: any[] = await DB.query(`
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE blocks.height = ${height}
`);
WHERE blocks.height = ?`,
[height]
);
if (rows.length <= 0) {
return null;
}
rows[0].fee_span = JSON.parse(rows[0].fee_span);
rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles);
return rows[0];
return await this.formatDbBlockIntoExtendedBlock(rows[0]);
} catch (e) {
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@@ -402,10 +422,7 @@ class BlocksRepository {
public async $getBlockByHash(hash: string): Promise<object | null> {
try {
const query = `
SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE hash = ?;
@@ -415,9 +432,8 @@ class BlocksRepository {
if (rows.length <= 0) {
return null;
}
rows[0].fee_span = JSON.parse(rows[0].fee_span);
return rows[0];
return await this.formatDbBlockIntoExtendedBlock(rows[0]);
} catch (e) {
logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@@ -833,6 +849,86 @@ class BlocksRepository {
throw e;
}
}
/**
* Convert a mysql row block into a BlockExtended. Note that you
* must provide the correct field into dbBlk object param
*
* @param dbBlk
*/
private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
const blk: Partial<BlockExtended> = {};
const extras: Partial<BlockExtension> = {};
// IEsploraApi.Block
blk.id = dbBlk.id;
blk.height = dbBlk.height;
blk.version = dbBlk.version;
blk.timestamp = dbBlk.timestamp;
blk.bits = dbBlk.bits;
blk.nonce = dbBlk.nonce;
blk.difficulty = dbBlk.difficulty;
blk.merkle_root = dbBlk.merkle_root;
blk.tx_count = dbBlk.tx_count;
blk.size = dbBlk.size;
blk.weight = dbBlk.weight;
blk.previousblockhash = dbBlk.previousblockhash;
blk.mediantime = dbBlk.mediantime;
// BlockExtension
extras.totalFees = dbBlk.totalFees;
extras.medianFee = dbBlk.medianFee;
extras.feeRange = JSON.parse(dbBlk.feeRange);
extras.reward = dbBlk.reward;
extras.pool = {
id: dbBlk.poolId,
name: dbBlk.poolName,
slug: dbBlk.poolSlug,
};
extras.avgFee = dbBlk.avgFee;
extras.avgFeeRate = dbBlk.avgFeeRate;
extras.coinbaseRaw = dbBlk.coinbaseRaw;
extras.coinbaseAddress = dbBlk.coinbaseAddress;
extras.coinbaseSignature = dbBlk.coinbaseSignature;
extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
extras.avgTxSize = dbBlk.avgTxSize;
extras.totalInputs = dbBlk.totalInputs;
extras.totalOutputs = dbBlk.totalOutputs;
extras.totalOutputAmt = dbBlk.totalOutputAmt;
extras.medianFeeAmt = dbBlk.medianFeeAmt;
extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
extras.segwitTotalSize = dbBlk.segwitTotalSize;
extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
extras.header = dbBlk.header,
extras.utxoSetChange = dbBlk.utxoSetChange;
extras.utxoSetSize = dbBlk.utxoSetSize;
extras.totalInputAmt = dbBlk.totalInputAmt;
extras.virtualSize = dbBlk.weight / 4.0;
// Re-org can happen after indexing so we need to always get the
// latest state from core
extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
// If we're missing block summary related field, check if we can populate them on the fly now
if (Common.blocksSummariesIndexingEnabled() &&
(extras.medianFeeAmt === null || extras.feePercentiles === null))
{
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
if (extras.feePercentiles === null) {
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
const summary = blocks.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
}
if (extras.feePercentiles !== null) {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
}
}
export default new BlocksRepository();