machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into download_assets_over_tor

Browse Source
This commit is contained in:
wiz
2022-04-14 21:35:59 +00:00
committed by GitHub
parent a5522c9861 fe5e685a38
commit d438d91377
100 changed files with 4903 additions and 1867 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/src/api/backend-info.ts
View File

@@ -2,6 +2,7 @@ import * as fs from 'fs';
import * as os from 'os';
import logger from '../logger';
import { IBackendInfo } from '../mempool.interfaces';
const { spawnSync } = require('child_process');
class BackendInfo {
private gitCommitHash = '';
@@ -27,10 +28,23 @@ class BackendInfo {
}
private setLatestCommitHash(): void {
try {
this.gitCommitHash = fs.readFileSync('../.git/refs/heads/master').toString().trim();
} catch (e) {
logger.err('Could not load git commit info: ' + (e instanceof Error ? e.message : e));
//TODO: share this logic with `generate-config.js`
if (process.env.DOCKER_COMMIT_HASH) {
this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
} else {
try {
const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
if (!gitRevParse.error) {
const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
this.gitCommitHash = output ? output : '?';
} else if (gitRevParse.error.code === 'ENOENT') {
console.log('git not found, cannot parse git hash');
this.gitCommitHash = '?';
}
} catch (e: any) {
console.log('Could not load git commit info: ' + e.message);
this.gitCommitHash = '?';
}
}
}

12
backend/src/api/bisq/markets-api.ts
View File

@@ -1,7 +1,7 @@
import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
import * as datetime from 'locutus/php/datetime';
const strtotime = require('./strtotime');
class BisqMarketsApi {
private cryptoCurrencyData: Currency[] = [];
@@ -312,7 +312,7 @@ class BisqMarketsApi {
getTickerFromMarket(market: string): Ticker | null {
let ticker: Ticker;
const timestamp_from = datetime.strtotime('-24 hour');
const timestamp_from = strtotime('-24 hour');
const timestamp_to = new Date().getTime() / 1000;
const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
@@ -638,13 +638,13 @@ class BisqMarketsApi {
case 'half_day':
return (ts - (ts % (3600 * 12)));
case 'day':
return datetime.strtotime('midnight today', ts);
return strtotime('midnight today', ts);
case 'week':
return datetime.strtotime('midnight sunday last week', ts);
return strtotime('midnight sunday last week', ts);
case 'month':
return datetime.strtotime('midnight first day of this month', ts);
return strtotime('midnight first day of this month', ts);
case 'year':
return datetime.strtotime('midnight first day of january', ts);
return strtotime('midnight first day of january', ts);
default:
throw new Error('Unsupported interval: ' + interval);
}

1375
backend/src/api/bisq/strtotime.ts Normal file
View File

File diff suppressed because it is too large Load Diff

90
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -25,7 +25,7 @@ class BitcoinApi implements AbstractBitcoinApi {
.then((transaction: IBitcoinApi.Transaction) => {
if (skipConversion) {
transaction.vout.forEach((vout) => {
vout.value = vout.value * 100000000;
vout.value = Math.round(vout.value * 100000000);
});
return transaction;
}
@@ -143,11 +143,11 @@ class BitcoinApi implements AbstractBitcoinApi {
esploraTransaction.vout = transaction.vout.map((vout) => {
return {
value: vout.value * 100000000,
value: Math.round(vout.value * 100000000),
scriptpubkey: vout.scriptPubKey.hex,
scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
: vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.asm) : '',
scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
};
});
@@ -157,7 +157,7 @@ class BitcoinApi implements AbstractBitcoinApi {
is_coinbase: !!vin.coinbase,
prevout: null,
scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.asm) || '',
scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
sequence: vin.sequence,
txid: vin.txid || '',
vout: vin.vout || 0,
@@ -212,6 +212,7 @@ class BitcoinApi implements AbstractBitcoinApi {
'witness_v0_scripthash': 'v0_p2wsh',
'witness_v1_taproot': 'v1_p2tr',
'nonstandard': 'nonstandard',
'multisig': 'multisig',
'nulldata': 'op_return'
};
@@ -235,7 +236,7 @@ class BitcoinApi implements AbstractBitcoinApi {
} else {
mempoolEntry = await this.$getMempoolEntry(transaction.txid);
}
transaction.fee = mempoolEntry.fees.base * 100000000;
transaction.fee = Math.round(mempoolEntry.fees.base * 100000000);
return transaction;
}
@@ -289,23 +290,68 @@ class BitcoinApi implements AbstractBitcoinApi {
return transaction;
}
private convertScriptSigAsm(str: string): string {
const a = str.split(' ');
private convertScriptSigAsm(hex: string): string {
const buf = Buffer.from(hex, 'hex');
const b: string[] = [];
a.forEach((chunk) => {
if (chunk.substr(0, 3) === 'OP_') {
chunk = chunk.replace(/^OP_(\d+)/, 'OP_PUSHNUM_$1');
chunk = chunk.replace('OP_CHECKSEQUENCEVERIFY', 'OP_CSV');
b.push(chunk);
} else {
chunk = chunk.replace('[ALL]', '01');
if (chunk === '0') {
b.push('OP_0');
let i = 0;
while (i < buf.length) {
const op = buf[i];
if (op >= 0x01 && op <= 0x4e) {
i++;
let push: number;
if (op === 0x4c) {
push = buf.readUInt8(i);
b.push('OP_PUSHDATA1');
i += 1;
} else if (op === 0x4d) {
push = buf.readUInt16LE(i);
b.push('OP_PUSHDATA2');
i += 2;
} else if (op === 0x4e) {
push = buf.readUInt32LE(i);
b.push('OP_PUSHDATA4');
i += 4;
} else {
b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk);
push = op;
b.push('OP_PUSHBYTES_' + push);
}
const data = buf.slice(i, i + push);
if (data.length !== push) {
break;
}
b.push(data.toString('hex'));
i += data.length;
} else {
if (op === 0x00) {
b.push('OP_0');
} else if (op === 0x4f) {
b.push('OP_PUSHNUM_NEG1');
} else if (op === 0xb1) {
b.push('OP_CLTV');
} else if (op === 0xb2) {
b.push('OP_CSV');
} else if (op === 0xba) {
b.push('OP_CHECKSIGADD');
} else {
const opcode = bitcoinjs.script.toASM([ op ]);
if (opcode && op < 0xfd) {
if (/^OP_(\d+)$/.test(opcode)) {
b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
} else {
b.push(opcode);
}
} else {
b.push('OP_RETURN_' + op);
}
}
i += 1;
}
});
}
return b.join(' ');
}
@@ -316,21 +362,21 @@ class BitcoinApi implements AbstractBitcoinApi {
if (vin.prevout.scriptpubkey_type === 'p2sh') {
const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
vin.inner_redeemscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(redeemScript, 'hex')));
vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
if (vin.witness && vin.witness.length > 2) {
const witnessScript = vin.witness[vin.witness.length - 1];
vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
}
if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
const witnessScript = vin.witness[vin.witness.length - 1];
vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) {
const witnessScript = vin.witness[vin.witness.length - 2];
vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
}

40
backend/src/api/blocks.ts
View File

@@ -23,6 +23,7 @@ class Blocks {
private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
private blockIndexingStarted = false;
public blockIndexingCompleted = false;
public reindexFlag = true; // Always re-index the latest indexed data in case the node went offline with an invalid block tip (reorg)
constructor() { }
@@ -74,9 +75,12 @@ class Blocks {
transactions.push(tx);
transactionsFetched++;
} catch (e) {
logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
if (i === 0) {
throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e);
logger.err(msg);
throw new Error(msg);
} else {
logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
}
}
}
@@ -135,9 +139,16 @@ class Blocks {
} else {
pool = await poolsRepository.$getUnknownPool();
}
if (!pool) { // We should never have this situation in practise
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. Check your "pools" table entries`);
return blockExtended;
}
blockExtended.extras.pool = {
id: pool.id,
name: pool.name
name: pool.name,
slug: pool.slug,
};
}
@@ -182,16 +193,19 @@ class Blocks {
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
public async $generateBlockDatabase() {
if (this.blockIndexingStarted) {
if (this.blockIndexingStarted && !this.reindexFlag) {
return;
}
this.reindexFlag = false;
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
return;
}
this.blockIndexingStarted = true;
this.blockIndexingCompleted = false;
try {
let currentBlockHeight = blockchainInfo.blocks;
@@ -203,11 +217,12 @@ class Blocks {
const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
const chunkSize = 10000;
let totaIndexed = await blocksRepository.$blockCount(null, null);
let indexedThisRun = 0;
let newlyIndexed = 0;
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
@@ -217,12 +232,11 @@ class Blocks {
const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
currentBlockHeight, endBlock);
if (missingBlockHeights.length <= 0) {
logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}`);
currentBlockHeight -= chunkSize;
continue;
}
logger.debug(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
for (const blockHeight of missingBlockHeights) {
if (blockHeight < lastBlockToIndex) {
@@ -244,14 +258,16 @@ class Blocks {
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
newlyIndexed++;
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
currentBlockHeight -= chunkSize;
}
logger.info('Block indexing completed');
logger.info(`Indexed ${newlyIndexed} blocks`);
} catch (e) {
logger.err('An error occured in $generateBlockDatabase(). Trying again later. ' + e);
logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
this.blockIndexingStarted = false;
return;
}
@@ -309,6 +325,12 @@ class Blocks {
if (Common.indexingEnabled()) {
await blocksRepository.$saveBlockInDatabase(blockExtended);
// If the last 10 blocks chain is not valid, re-index them (reorg)
const chainValid = await blocksRepository.$validateRecentBlocks();
if (!chainValid) {
this.reindexFlag = true;
}
}
if (block.height % 2016 === 0) {

163
backend/src/api/database-migration.ts
View File

@@ -1,12 +1,11 @@
import { PoolConnection } from 'mysql2/promise';
import config from '../config';
import { DB } from '../database';
import DB from '../database';
import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 15;
private static currentVersion = 17;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@@ -77,107 +76,112 @@ class DatabaseMigration {
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
const connection = await DB.getConnection();
try {
await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(connection, this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
await this.$executeQuery(connection, `CREATE INDEX added ON statistics (added);`);
await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
}
if (databaseSchemaVersion < 3) {
await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
}
if (databaseSchemaVersion < 4) {
await this.$executeQuery(connection, 'DROP table IF EXISTS blocks;');
await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
await this.$executeQuery('DROP table IF EXISTS blocks;');
await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
// We also fix the pools.id type so we need to drop/re-create the foreign key
await this.$executeQuery(connection, 'ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
await this.$executeQuery(connection, 'ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
// Add new block indexing fields
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
}
if (databaseSchemaVersion < 7 && isBitcoin === true) {
await this.$executeQuery(connection, 'DROP table IF EXISTS hashrates;');
await this.$executeQuery(connection, this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
await this.$executeQuery('DROP table IF EXISTS hashrates;');
await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
}
if (databaseSchemaVersion < 8 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
if (databaseSchemaVersion < 9 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
}
if (databaseSchemaVersion < 10 && isBitcoin === true) {
await this.$executeQuery(connection, 'ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
}
if (databaseSchemaVersion < 11 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, `ALTER TABLE blocks
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(`ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL,
ADD avg_fee_rate INT UNSIGNED NULL
`);
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 12 && isBitcoin === true) {
// No need to re-index because the new data type can contain larger values
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 13 && isBitcoin === true) {
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 14 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
}
connection.release();
if (databaseSchemaVersion < 16 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
}
if (databaseSchemaVersion < 17 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
}
} catch (e) {
connection.release();
throw e;
}
}
@@ -194,13 +198,11 @@ class DatabaseMigration {
return;
}
const connection = await DB.getConnection();
try {
// We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X
const query = `SELECT COUNT(1) hasIndex FROM INFORMATION_SCHEMA.STATISTICS
WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`;
const [rows] = await this.$executeQuery(connection, query, true);
const [rows] = await this.$executeQuery(query, true);
if (rows[0].hasIndex === 0) {
logger.debug('MIGRATIONS: `statistics.added` is not indexed');
this.statisticsAddedIndexed = false;
@@ -214,28 +216,24 @@ class DatabaseMigration {
logger.err('MIGRATIONS: Unable to check if `statistics.added` INDEX exist or not.');
this.statisticsAddedIndexed = true;
}
connection.release();
}
/**
* Small query execution wrapper to log all executed queries
*/
private async $executeQuery(connection: PoolConnection, query: string, silent: boolean = false): Promise<any> {
private async $executeQuery(query: string, silent: boolean = false): Promise<any> {
if (!silent) {
logger.debug('MIGRATIONS: Execute query:\n' + query);
}
return connection.query<any>({ sql: query, timeout: this.queryTimeout });
return DB.query({ sql: query, timeout: this.queryTimeout });
}
/**
* Check if 'table' exists in the database
*/
private async $checkIfTableExists(table: string): Promise<boolean> {
const connection = await DB.getConnection();
const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return rows[0]['COUNT(*)'] === 1;
}
@@ -243,10 +241,8 @@ class DatabaseMigration {
* Get current database version
*/
private async $getSchemaVersionFromDatabase(): Promise<number> {
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = 'schema_version';`;
const [rows] = await this.$executeQuery(connection, query, true);
connection.release();
const [rows] = await this.$executeQuery(query, true);
return rows[0]['number'];
}
@@ -254,8 +250,6 @@ class DatabaseMigration {
* Create the `state` table
*/
private async $createMigrationStateTable(): Promise<void> {
const connection = await DB.getConnection();
try {
const query = `CREATE TABLE IF NOT EXISTS state (
name varchar(25) NOT NULL,
@@ -263,15 +257,12 @@ class DatabaseMigration {
string varchar(100) NULL,
CONSTRAINT name_unique UNIQUE (name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
await this.$executeQuery(connection, query);
await this.$executeQuery(query);
// Set initial values
await this.$executeQuery(connection, `INSERT INTO state VALUES('schema_version', 0, NULL);`);
await this.$executeQuery(connection, `INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
connection.release();
await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
} catch (e) {
connection.release();
throw e;
}
}
@@ -286,18 +277,14 @@ class DatabaseMigration {
}
transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
const connection = await DB.getConnection();
try {
await this.$executeQuery(connection, 'START TRANSACTION;');
await this.$executeQuery('START TRANSACTION;');
for (const query of transactionQueries) {
await this.$executeQuery(connection, query);
await this.$executeQuery(query);
}
await this.$executeQuery(connection, 'COMMIT;');
connection.release();
await this.$executeQuery('COMMIT;');
} catch (e) {
await this.$executeQuery(connection, 'ROLLBACK;');
connection.release();
await this.$executeQuery('ROLLBACK;');
throw e;
}
}
@@ -337,14 +324,12 @@ class DatabaseMigration {
* Print current database version
*/
private async $printDatabaseVersion() {
const connection = await DB.getConnection();
try {
const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true);
const [rows] = await this.$executeQuery('SELECT VERSION() as version;', true);
logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`);
} catch (e) {
logger.debug(`MIGRATIONS: Could not fetch database engine version. ` + e);
}
connection.release();
}
// Couple of wrappers to clean the main logic
@@ -481,24 +466,22 @@ class DatabaseMigration {
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates'];
const connection = await DB.getConnection();
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {
logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
continue;
};
}
await this.$executeQuery(connection, `TRUNCATE ${table}`, true);
await this.$executeQuery(`TRUNCATE ${table}`, true);
if (table === 'hashrates') {
await this.$executeQuery(connection, 'UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
}
logger.notice(`Table ${table} has been truncated`);
}
} catch (e) {
logger.warn(`Unable to erase indexed data`);
}
connection.release();
}
}

18
backend/src/api/liquid/elements-parser.ts
View File

@@ -2,7 +2,7 @@ import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface';
import bitcoinClient from '../bitcoin/bitcoin-client';
import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
import { Common } from '../common';
import { DB } from '../../database';
import DB from '../../database';
import logger from '../../logger';
class ElementsParser {
@@ -33,10 +33,8 @@ class ElementsParser {
}
public async $getPegDataByMonth(): Promise<any> {
const connection = await DB.getConnection();
const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
const [rows] = await connection.query<any>(query);
connection.release();
const [rows] = await DB.query(query);
return rows;
}
@@ -79,7 +77,6 @@ class ElementsParser {
protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
const connection = await DB.getConnection();
const query = `INSERT INTO elements_pegs(
block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -87,24 +84,19 @@ class ElementsParser {
const params: (string | number)[] = [
height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
];
await connection.query(query, params);
connection.release();
await DB.query(query, params);
logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
}
protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
const [rows] = await connection.query<any>(query);
connection.release();
const [rows] = await DB.query(query);
return rows[0]['number'];
}
protected async $saveLatestBlockToDatabase(blockHeight: number) {
const connection = await DB.getConnection();
const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
await connection.query<any>(query, [blockHeight]);
connection.release();
await DB.query(query, [blockHeight]);
}
}

179
backend/src/api/mining.ts
View File

@@ -5,6 +5,7 @@ import HashratesRepository from '../repositories/HashratesRepository';
import bitcoinClient from './bitcoin/bitcoin-client';
import logger from '../logger';
import blocks from './blocks';
import { Common } from './common';
class Mining {
hashrateIndexingStarted = false;
@@ -13,6 +14,26 @@ class Mining {
constructor() {
}
/**
* Get historical block reward and total fee
*/
public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockFees(
this.getTimeRange(interval),
Common.getSqlInterval(interval)
);
}
/**
* Get historical block rewards
*/
public async $getHistoricalBlockRewards(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockRewards(
this.getTimeRange(interval),
Common.getSqlInterval(interval)
);
}
/**
* Generate high level overview of the pool ranks and general stats
*/
@@ -33,7 +54,8 @@ class Mining {
link: poolInfo.link,
blockCount: poolInfo.blockCount,
rank: rank++,
emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0
emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
slug: poolInfo.slug,
};
poolsStats.push(poolStat);
});
@@ -44,8 +66,8 @@ class Mining {
const blockCount: number = await BlocksRepository.$blockCount(null, interval);
poolsStatistics['blockCount'] = blockCount;
const blockHeightTip = await bitcoinClient.getBlockCount();
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(144, blockHeightTip);
const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
return poolsStatistics;
@@ -54,19 +76,37 @@ class Mining {
/**
* Get all mining pool stats for a pool
*/
public async $getPoolStat(poolId: number): Promise<object> {
const pool = await PoolsRepository.$getPool(poolId);
public async $getPoolStat(slug: string): Promise<object> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error(`This mining pool does not exist`);
}
const blockCount: number = await BlocksRepository.$blockCount(poolId);
const emptyBlocksCount = await BlocksRepository.$countEmptyBlocks(poolId);
const blockCount: number = await BlocksRepository.$blockCount(pool.id);
const totalBlock: number = await BlocksRepository.$blockCount(null, null);
const blockCount24h: number = await BlocksRepository.$blockCount(pool.id, '24h');
const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
const currentEstimatedkHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
return {
pool: pool,
blockCount: blockCount,
emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
blockCount: {
'all': blockCount,
'24h': blockCount24h,
'1w': blockCount1w,
},
blockShare: {
'all': blockCount / totalBlock,
'24h': blockCount24h / totalBlock24h,
'1w': blockCount1w / totalBlock1w,
},
estimatedHashrate: currentEstimatedkHashrate * (blockCount24h / totalBlock24h),
reportedHashrate: null,
};
}
@@ -85,37 +125,43 @@ class Mining {
return;
}
// We only run this once a week
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing');
const now = new Date();
if ((now.getTime() / 1000) - latestTimestamp < 604800) {
return;
}
try {
this.weeklyHashrateIndexingStarted = true;
logger.info(`Indexing mining pools weekly hashrates`);
// We only run this once a week
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing') * 1000;
if (now.getTime() - latestTimestamp < 604800000) {
this.weeklyHashrateIndexingStarted = false;
return;
}
} catch (e) {
this.weeklyHashrateIndexingStarted = false;
throw e;
}
try {
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
const hashrates: any[] = [];
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
const lastMondayMidnight = this.getDateMidnight(lastMonday);
let toTimestamp = Math.round((lastMondayMidnight.getTime() - 604800) / 1000);
let toTimestamp = lastMondayMidnight.getTime();
const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008;
let indexedThisRun = 0;
let totalIndexed = 0;
let startedAt = new Date().getTime() / 1000;
let newlyIndexed = 0;
let startedAt = new Date().getTime();
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 604800;
const fromTimestamp = toTimestamp - 604800000;
// Skip already indexed weeks
if (indexedTimestamp.includes(toTimestamp)) {
toTimestamp -= 604800;
if (indexedTimestamp.includes(toTimestamp / 1000)) {
toTimestamp -= 604800000;
++totalIndexed;
continue;
}
@@ -123,17 +169,17 @@ class Mining {
// Check if we have blocks for the previous week (which mean that the week
// we are currently indexing has complete data)
const blockStatsPreviousWeek: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp - 604800, toTimestamp - 604800);
null, (fromTimestamp - 604800000) / 1000, (toTimestamp - 604800000) / 1000);
if (blockStatsPreviousWeek.blockCount === 0) { // We are done indexing
break;
}
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp, toTimestamp);
null, fromTimestamp / 1000, toTimestamp / 1000);
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp, toTimestamp);
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
pools = pools.map((pool: any) => {
pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
@@ -143,7 +189,7 @@ class Mining {
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: toTimestamp,
hashrateTimestamp: toTimestamp / 1000,
avgHashrate: pool['hashrate'],
poolId: pool.poolId,
share: pool['share'],
@@ -151,26 +197,29 @@ class Mining {
});
}
newlyIndexed += hashrates.length;
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime()) - startedAt)) / 1000;
if (elapsedSeconds > 1) {
const weeksPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
const formattedDate = new Date(fromTimestamp).toUTCString();
const weeksLeft = Math.round(totalWeekIndexed - totalIndexed);
logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds} weeks/sec | ~${weeksLeft} weeks left to index`);
startedAt = new Date().getTime() / 1000;
startedAt = new Date().getTime();
indexedThisRun = 0;
}
toTimestamp -= 604800;
toTimestamp -= 604800000;
++indexedThisRun;
++totalIndexed;
}
this.weeklyHashrateIndexingStarted = false;
await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing');
logger.info(`Weekly pools hashrate indexing completed`);
if (newlyIndexed > 0) {
logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
}
} catch (e) {
this.weeklyHashrateIndexingStarted = false;
throw e;
@@ -185,35 +234,41 @@ class Mining {
return;
}
// We only run this once a day
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_hashrates_indexing');
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 86400) {
return;
}
const now = new Date().getTime();
try {
this.hashrateIndexingStarted = true;
logger.info(`Indexing network daily hashrate`);
// We only run this once a day
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_hashrates_indexing') * 1000;
if (now - latestTimestamp < 86400000) {
this.hashrateIndexingStarted = false;
return;
}
} catch (e) {
this.hashrateIndexingStarted = false;
throw e;
}
try {
const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
let toTimestamp = Math.round(lastMidnight.getTime());
const hashrates: any[] = [];
const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
let indexedThisRun = 0;
let totalIndexed = 0;
let startedAt = new Date().getTime() / 1000;
let newlyIndexed = 0;
let startedAt = new Date().getTime();
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 86400;
const fromTimestamp = toTimestamp - 86400000;
// Skip already indexed weeks
if (indexedTimestamp.includes(toTimestamp)) {
toTimestamp -= 86400;
if (indexedTimestamp.includes(toTimestamp / 1000)) {
toTimestamp -= 86400000;
++totalIndexed;
continue;
}
@@ -221,18 +276,18 @@ class Mining {
// Check if we have blocks for the previous day (which mean that the day
// we are currently indexing has complete data)
const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp - 86400, toTimestamp - 86400);
null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
break;
}
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp, toTimestamp);
null, fromTimestamp / 1000, toTimestamp / 1000);
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
hashrates.push({
hashrateTimestamp: toTimestamp,
hashrateTimestamp: toTimestamp / 1000,
avgHashrate: lastBlockHashrate,
poolId: 0,
share: 1,
@@ -240,21 +295,23 @@ class Mining {
});
if (hashrates.length > 10) {
newlyIndexed += hashrates.length;
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
}
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const elapsedSeconds = Math.max(1, Math.round(new Date().getTime() - startedAt)) / 1000;
if (elapsedSeconds > 1) {
const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
const formattedDate = new Date(fromTimestamp).toUTCString();
const daysLeft = Math.round(totalDayIndexed - totalIndexed);
logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`);
startedAt = new Date().getTime() / 1000;
logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ` +
`~${daysLeft} days left to index`);
startedAt = new Date().getTime();
indexedThisRun = 0;
}
toTimestamp -= 86400;
toTimestamp -= 86400000;
++indexedThisRun;
++totalIndexed;
}
@@ -269,11 +326,14 @@ class Mining {
});
}
newlyIndexed += hashrates.length;
await HashratesRepository.$saveHashrates(hashrates);
await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing');
this.hashrateIndexingStarted = false;
logger.info(`Daily network hashrate indexing completed`);
if (newlyIndexed > 0) {
logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
}
} catch (e) {
this.hashrateIndexingStarted = false;
throw e;
@@ -288,6 +348,21 @@ class Mining {
return date;
}
private getTimeRange(interval: string | null): number {
switch (interval) {
case '3y': return 43200; // 12h
case '2y': return 28800; // 8h
case '1y': return 28800; // 8h
case '6m': return 10800; // 3h
case '3m': return 7200; // 2h
case '1m': return 1800; // 30min
case '1w': return 300; // 5min
case '3d': return 1;
case '24h': return 1;
default: return 86400; // 24h
}
}
}
export default new Mining();

82
backend/src/api/pools-parser.ts
View File

@@ -1,5 +1,4 @@
import { readFileSync } from 'fs';
import { DB } from '../database';
import DB from '../database';
import logger from '../logger';
import config from '../config';
@@ -8,29 +7,20 @@ interface Pool {
link: string;
regexes: string[];
addresses: string[];
slug: string;
}
class PoolsParser {
slugWarnFlag = false;
/**
* Parse the pools.json file, consolidate the data and dump it into the database
*/
public async migratePoolsJson() {
public async migratePoolsJson(poolsJson: object) {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
logger.debug('Importing pools.json to the database, open ./pools.json');
let poolsJson: object = {};
try {
const fileContent: string = readFileSync('./pools.json', 'utf8');
poolsJson = JSON.parse(fileContent);
} catch (e) {
logger.err('Unable to open ./pools.json, does the file exist?');
await this.insertUnknownPool();
return;
}
// First we save every entries without paying attention to pool duplication
const poolsDuplicated: Pool[] = [];
@@ -42,6 +32,7 @@ class PoolsParser {
'link': (<Pool>coinbaseTags[i][1]).link,
'regexes': [coinbaseTags[i][0]],
'addresses': [],
'slug': ''
});
}
logger.debug('Parse payout_addresses');
@@ -52,6 +43,7 @@ class PoolsParser {
'link': (<Pool>addressesTags[i][1]).link,
'regexes': [],
'addresses': [addressesTags[i][0]],
'slug': ''
});
}
@@ -66,13 +58,11 @@ class PoolsParser {
logger.debug(`Found ${poolNames.length} unique mining pools`);
// Get existing pools from the db
const connection = await DB.getConnection();
let existingPools;
try {
[existingPools] = await connection.query<any>({ sql: 'SELECT * FROM pools;', timeout: 120000 });
[existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
} catch (e) {
logger.err('Unable to get existing pools from the database, skipping pools.json import');
connection.release();
logger.err('Cannot get existing pools from the database, skipping pools.json import');
return;
}
@@ -91,13 +81,29 @@ class PoolsParser {
const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
let slug: string | undefined;
try {
slug = poolsJson['slugs'][poolNames[i]];
} catch (e) {
if (this.slugWarnFlag === false) {
logger.warn(`pools.json does not seem to contain the 'slugs' object`);
this.slugWarnFlag = true;
}
}
if (slug === undefined) {
// Only keep alphanumerical
slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
}
if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
logger.debug(`Update '${finalPoolName}' mining pool`);
finalPoolDataUpdate.push({
'name': finalPoolName,
'link': match[0].link,
'regexes': allRegexes,
'addresses': allAddresses,
'slug': slug
});
} else {
logger.debug(`Add '${finalPoolName}' mining pool`);
@@ -106,6 +112,7 @@ class PoolsParser {
'link': match[0].link,
'regexes': allRegexes,
'addresses': allAddresses,
'slug': slug
});
}
}
@@ -113,10 +120,11 @@ class PoolsParser {
logger.debug(`Update pools table now`);
// Add new mining pools into the database
let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES ';
let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
for (let i = 0; i < finalPoolDataAdd.length; ++i) {
queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
'${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}'),`;
'${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
}
queryAdd = queryAdd.slice(0, -1) + ';';
@@ -126,24 +134,23 @@ class PoolsParser {
updateQueries.push(`
UPDATE pools
SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}'
regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
slug='${finalPoolDataUpdate[i].slug}'
WHERE name='${finalPoolDataUpdate[i].name}'
;`);
}
try {
if (finalPoolDataAdd.length > 0) {
await connection.query<any>({ sql: queryAdd, timeout: 120000 });
await DB.query({ sql: queryAdd, timeout: 120000 });
}
for (const query of updateQueries) {
await connection.query<any>({ sql: query, timeout: 120000 });
await DB.query({ sql: query, timeout: 120000 });
}
await this.insertUnknownPool();
connection.release();
logger.info('Mining pools.json import completed');
} catch (e) {
connection.release();
logger.err(`Unable to import pools in the database!`);
logger.err(`Cannot import pools in the database`);
throw e;
}
}
@@ -152,21 +159,24 @@ class PoolsParser {
* Manually add the 'unknown pool'
*/
private async insertUnknownPool() {
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
if (rows.length === 0) {
logger.debug('Manually inserting "Unknown" mining pool into the databse');
await connection.query({
sql: `INSERT INTO pools(name, link, regexes, addresses)
VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]");
await DB.query({
sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
`});
} else {
await DB.query(`UPDATE pools
SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
regexes='[]', addresses='[]',
slug='unknown'
WHERE name='Unknown'
`);
}
} catch (e) {
logger.err('Unable to insert "Unknown" mining pool');
}
connection.release();
}
}

76
backend/src/api/statistics.ts
View File

@@ -1,5 +1,5 @@
import memPool from './mempool';
import { DB } from '../database';
import DB from '../database';
import logger from '../logger';
import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces';
@@ -155,7 +155,6 @@ class Statistics {
}
private async $createZeroedStatistic(): Promise<number | undefined> {
const connection = await DB.getConnection();
try {
const query = `INSERT INTO statistics(
added,
@@ -206,17 +205,14 @@ class Statistics {
)
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
const [result]: any = await connection.query(query);
connection.release();
const [result]: any = await DB.query(query);
return result.insertId;
} catch (e) {
connection.release();
logger.err('$create() error' + (e instanceof Error ? e.message : e));
}
}
private async $create(statistics: Statistic): Promise<number | undefined> {
const connection = await DB.getConnection();
try {
const query = `INSERT INTO statistics(
added,
@@ -314,11 +310,9 @@ class Statistics {
statistics.vsize_1800,
statistics.vsize_2000,
];
const [result]: any = await connection.query(query, params);
connection.release();
const [result]: any = await DB.query(query, params);
return result.insertId;
} catch (e) {
connection.release();
logger.err('$create() error' + (e instanceof Error ? e.message : e));
}
}
@@ -421,10 +415,8 @@ class Statistics {
private async $get(id: number): Promise<OptimizedStatistic | undefined> {
try {
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
const [rows] = await connection.query<any>(query, [id]);
connection.release();
const [rows] = await DB.query(query, [id]);
if (rows[0]) {
return this.mapStatisticToOptimizedStatistic([rows[0]])[0];
}
@@ -435,11 +427,9 @@ class Statistics {
public async $list2H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list2H() error' + (e instanceof Error ? e.message : e));
return [];
@@ -448,11 +438,9 @@ class Statistics {
public async $list24H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list24h() error' + (e instanceof Error ? e.message : e));
return [];
@@ -461,11 +449,9 @@ class Statistics {
public async $list1W(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list1W() error' + (e instanceof Error ? e.message : e));
return [];
@@ -474,11 +460,9 @@ class Statistics {
public async $list1M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list1M() error' + (e instanceof Error ? e.message : e));
return [];
@@ -487,11 +471,9 @@ class Statistics {
public async $list3M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list3M() error' + (e instanceof Error ? e.message : e));
return [];
@@ -500,11 +482,9 @@ class Statistics {
public async $list6M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list6M() error' + (e instanceof Error ? e.message : e));
return [];
@@ -513,11 +493,9 @@ class Statistics {
public async $list1Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list1Y() error' + (e instanceof Error ? e.message : e));
return [];
@@ -526,11 +504,9 @@ class Statistics {
public async $list2Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = this.getQueryForDays(28800, "2 YEAR"); // 8h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const query = this.getQueryForDays(28800, '2 YEAR'); // 8h interval
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list2Y() error' + (e instanceof Error ? e.message : e));
return [];
@@ -539,11 +515,9 @@ class Statistics {
public async $list3Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.getConnection();
const query = this.getQueryForDays(43200, "3 YEAR"); // 12h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
return this.mapStatisticToOptimizedStatistic(rows);
const query = this.getQueryForDays(43200, '3 YEAR'); // 12h interval
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list3Y() error' + (e instanceof Error ? e.message : e));
return [];

76
backend/src/database.ts
View File

@@ -1,51 +1,51 @@
import config from './config';
import { createPool, PoolConnection } from 'mysql2/promise';
import { createPool, Pool, PoolConnection } from 'mysql2/promise';
import logger from './logger';
import { PoolOptions } from 'mysql2/typings/mysql';
export class DB {
static poolConfig = ():PoolOptions => {
let poolConfig:PoolOptions = {
port: config.DATABASE.PORT,
database: config.DATABASE.DATABASE,
user: config.DATABASE.USERNAME,
password: config.DATABASE.PASSWORD,
connectionLimit: 10,
supportBigNumbers: true,
timezone: '+00:00',
}
if (config.DATABASE.SOCKET !== "") {
poolConfig.socketPath = config.DATABASE.SOCKET;
class DB {
constructor() {
if (config.DATABASE.SOCKET !== '') {
this.poolConfig.socketPath = config.DATABASE.SOCKET;
} else {
poolConfig.host = config.DATABASE.HOST;
this.poolConfig.host = config.DATABASE.HOST;
}
return poolConfig;
}
static pool = createPool(DB.poolConfig());
private pool: Pool | null = null;
private poolConfig: PoolOptions = {
port: config.DATABASE.PORT,
database: config.DATABASE.DATABASE,
user: config.DATABASE.USERNAME,
password: config.DATABASE.PASSWORD,
connectionLimit: 10,
supportBigNumbers: true,
timezone: '+00:00',
};
static connectionsReady: number[] = [];
public async query(query, params?) {
const pool = await this.getPool();
return pool.query(query, params);
}
static async getConnection() {
const connection: PoolConnection = await DB.pool.getConnection();
const connectionId = connection['connection'].connectionId;
if (!DB.connectionsReady.includes(connectionId)) {
await connection.query(`SET time_zone='+00:00';`);
this.connectionsReady.push(connectionId);
public async checkDbConnection() {
try {
await this.query('SELECT ?', [1]);
logger.info('Database connection established.');
} catch (e) {
logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
process.exit(1);
}
return connection;
}
private async getPool(): Promise<Pool> {
if (this.pool === null) {
this.pool = createPool(this.poolConfig);
this.pool.on('connection', function (newConnection: PoolConnection) {
newConnection.query(`SET time_zone='+00:00'`);
});
}
return this.pool;
}
}
export async function checkDbConnection() {
try {
const connection = await DB.getConnection();
logger.info('Database connection established.');
connection.release();
} catch (e) {
logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
process.exit(1);
}
}
export default new DB();

41
backend/src/index.ts
View File

@@ -5,7 +5,7 @@ import * as WebSocket from 'ws';
import * as cluster from 'cluster';
import axios from 'axios';
import { checkDbConnection, DB } from './database';
import DB from './database';
import config from './config';
import routes from './routes';
import blocks from './api/blocks';
@@ -22,12 +22,13 @@ import loadingIndicators from './api/loading-indicators';
import mempool from './api/mempool';
import elementsParser from './api/liquid/elements-parser';
import databaseMigration from './api/database-migration';
import poolsParser from './api/pools-parser';
import syncAssets from './sync-assets';
import icons from './api/liquid/icons';
import { Common } from './api/common';
import mining from './api/mining';
import HashratesRepository from './repositories/HashratesRepository';
import BlocksRepository from './repositories/BlocksRepository';
import poolsUpdater from './tasks/pools-updater';
class Server {
private wss: WebSocket.Server | undefined;
@@ -88,18 +89,17 @@ class Server {
diskCache.loadMempoolCache();
if (config.DATABASE.ENABLED) {
await checkDbConnection();
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex != undefined) { // Re-index requests
const tables = process.env.npm_config_reindex.split(',');
logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds from now (using '--reindex') ...`);
logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
await Common.sleep(5000);
await databaseMigration.$truncateIndexedData(tables);
}
await databaseMigration.$initializeOrMigrateDatabase();
if (Common.indexingEnabled()) {
await this.$resetHashratesIndexingState();
await poolsParser.migratePoolsJson();
}
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
@@ -169,8 +169,12 @@ class Server {
}
async $resetHashratesIndexingState() {
await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing', 0);
try {
await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
}
}
async $runIndexingWhenReady() {
@@ -179,11 +183,16 @@ class Server {
}
try {
blocks.$generateBlockDatabase();
await poolsUpdater.updatePoolsJson();
if (blocks.reindexFlag) {
await BlocksRepository.$deleteBlocks(10);
await HashratesRepository.$deleteLastEntries();
}
await blocks.$generateBlockDatabase();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
} catch (e) {
logger.err(`Unable to run indexing right now, trying again later. ` + e);
logger.err(`Indexing failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
}
}
@@ -301,18 +310,18 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/2y', routes.$getPools.bind(routes, '2y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3y', routes.$getPools.bind(routes, '3y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/all', routes.$getPools.bind(routes, 'all'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/hashrate', routes.$getPoolHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/blocks', routes.$getPoolBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/blocks/:height', routes.$getPoolBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId', routes.$getPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/:interval', routes.$getPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty', routes.$getHistoricalDifficulty)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty/:interval', routes.$getHistoricalDifficulty)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', routes.$getPoolHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', routes.$getPoolBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', routes.$getPoolBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/:interval', routes.$getPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools', routes.$getPoolsHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate', routes.$getHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
;
}

3
backend/src/mempool.interfaces.ts
View File

@@ -6,6 +6,7 @@ export interface PoolTag {
link: string;
regexes: string; // JSON array
addresses: string; // JSON array
slug: string;
}
export interface PoolInfo {
@@ -13,6 +14,7 @@ export interface PoolInfo {
name: string;
link: string;
blockCount: number;
slug: string;
}
export interface PoolStats extends PoolInfo {
@@ -87,6 +89,7 @@ export interface BlockExtension {
pool?: {
id: number;
name: string;
slug: string;
};
avgFee?: number;
avgFeeRate?: number;

206
backend/src/repositories/BlocksRepository.ts
View File

@@ -1,16 +1,15 @@
import { BlockExtended, PoolTag } from '../mempool.interfaces';
import { DB } from '../database';
import { BlockExtended } from '../mempool.interfaces';
import DB from '../database';
import logger from '../logger';
import { Common } from '../api/common';
import { prepareBlock } from '../utils/blocks-utils';
import PoolsRepository from './PoolsRepository';
class BlocksRepository {
/**
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(block: BlockExtended) {
const connection = await DB.getConnection();
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
@@ -49,15 +48,12 @@ class BlocksRepository {
block.extras.avgFeeRate,
];
await connection.query(query, params);
connection.release();
await DB.query(query, params);
} catch (e: any) {
connection.release();
if (e.errno === 1062) { // ER_DUP_ENTRY
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
} else {
connection.release();
logger.err('$saveBlockInDatabase() error: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -71,15 +67,13 @@ class BlocksRepository {
return [];
}
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(`
const [rows]: any[] = await DB.query(`
SELECT height
FROM blocks
WHERE height <= ? AND height >= ?
ORDER BY height DESC;
`, [startHeight, endHeight]);
connection.release();
const indexedBlockHeights: number[] = [];
rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
@@ -88,8 +82,7 @@ class BlocksRepository {
return missingBlocksHeights;
} catch (e) {
connection.release();
logger.err('$getMissingBlocksBetweenHeights() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot retrieve blocks list to index. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -117,15 +110,11 @@ class BlocksRepository {
query += ` GROUP by pools.id`;
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
const [rows] = await DB.query(query, params);
return rows;
} catch (e) {
connection.release();
logger.err('$getEmptyBlocks() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot count empty blocks. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -154,15 +143,11 @@ class BlocksRepository {
query += ` blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
const [rows] = await DB.query(query, params);
return <number>rows[0].blockCount;
} catch (e) {
connection.release();
logger.err('$blockCount() error' + (e instanceof Error ? e.message : e));
logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -193,15 +178,11 @@ class BlocksRepository {
}
query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`;
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
const [rows] = await DB.query(query, params);
return <number>rows[0];
} catch (e) {
connection.release();
logger.err('$blockCountBetweenTimestamp() error' + (e instanceof Error ? e.message : e));
logger.err(`Cannot count blocks for this pool (using timestamps). Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -215,10 +196,8 @@ class BlocksRepository {
ORDER BY height
LIMIT 1;`;
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(query);
connection.release();
const [rows]: any[] = await DB.query(query);
if (rows.length <= 0) {
return -1;
@@ -226,8 +205,7 @@ class BlocksRepository {
return <number>rows[0].blockTimestamp;
} catch (e) {
connection.release();
logger.err('$oldestBlockTimestamp() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot get oldest indexed block timestamp. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -235,13 +213,18 @@ class BlocksRepository {
/**
* Get blocks mined by a specific mining pool
*/
public async $getBlocksByPool(poolId: number, startHeight: number | undefined = undefined): Promise<object[]> {
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error(`This mining pool does not exist`);
}
const params: any[] = [];
let query = ` SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
previous_block_hash as previousblockhash
FROM blocks
WHERE pool_id = ?`;
params.push(poolId);
params.push(pool.id);
if (startHeight !== undefined) {
query += ` AND height < ?`;
@@ -251,20 +234,17 @@ class BlocksRepository {
query += ` ORDER BY height DESC
LIMIT 10`;
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
const [rows] = await DB.query(query, params);
const blocks: BlockExtended[] = [];
for (let block of <object[]>rows) {
for (const block of <object[]>rows) {
blocks.push(prepareBlock(block));
}
return blocks;
} catch (e) {
connection.release();
logger.err('$getBlocksByPool() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot get blocks for this pool. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -273,18 +253,16 @@ class BlocksRepository {
* Get one block by height
*/
public async $getBlockByHeight(height: number): Promise<object | null> {
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(`
const [rows]: any[] = await DB.query(`
SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE height = ${height};
`);
connection.release();
if (rows.length <= 0) {
return null;
@@ -292,8 +270,7 @@ class BlocksRepository {
return rows[0];
} catch (e) {
connection.release();
logger.err('$getBlockByHeight() error' + (e instanceof Error ? e.message : e));
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -304,8 +281,6 @@ class BlocksRepository {
public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
interval = Common.getSqlInterval(interval);
const connection = await DB.getConnection();
// :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
// Basically, using temporary user defined fields, we are able to extract all
// difficulty adjustments from the blocks tables.
@@ -339,34 +314,15 @@ class BlocksRepository {
`;
try {
const [rows]: any[] = await connection.query(query);
connection.release();
const [rows]: any[] = await DB.query(query);
for (let row of rows) {
for (const row of rows) {
delete row['rn'];
}
return rows;
} catch (e) {
connection.release();
logger.err('$getBlocksDifficulty() error' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Return oldest blocks height
*/
public async $getOldestIndexedBlockHeight(): Promise<number> {
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(`SELECT MIN(height) as minHeight FROM blocks`);
connection.release();
return rows[0].minHeight;
} catch (e) {
connection.release();
logger.err('$getOldestIndexedBlockHeight() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -375,21 +331,99 @@ class BlocksRepository {
* Get general block stats
*/
public async $getBlockStats(blockCount: number): Promise<any> {
let connection;
try {
connection = await DB.getConnection();
// We need to use a subquery
const query = `SELECT SUM(reward) as totalReward, SUM(fees) as totalFee, SUM(tx_count) as totalTx
FROM (SELECT reward, fees, tx_count FROM blocks ORDER by height DESC LIMIT ${blockCount}) as sub`;
const query = `
SELECT MIN(height) as startBlock, MAX(height) as endBlock, SUM(reward) as totalReward, SUM(fees) as totalFee, SUM(tx_count) as totalTx
FROM
(SELECT height, reward, fees, tx_count FROM blocks
ORDER by height DESC
LIMIT ?) as sub`;
const [rows]: any = await DB.query(query, [blockCount]);
const [rows]: any = await connection.query(query);
connection.release();
return rows[0];
} catch (e) {
connection.release();
logger.err('$getBlockStats() error: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot generate reward stats. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/*
* Check if the last 10 blocks chain is valid
*/
public async $validateRecentBlocks(): Promise<boolean> {
try {
const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
for (let i = 0; i < lastBlocks.length - 1; ++i) {
if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
return false;
}
}
return true;
} catch (e) {
return true; // Don't do anything if there is a db error
}
}
/**
* Delete $count blocks from the database
*/
public async $deleteBlocks(count: number) {
logger.info(`Delete ${count} most recent indexed blocks from the database`);
try {
await DB.query(`DELETE FROM blocks ORDER BY height DESC LIMIT ${count};`);
} catch (e) {
logger.err('Cannot delete recent indexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Get the historical averaged block fees
*/
public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(fees) as INT) as avg_fees
FROM blocks`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot generate block fees history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get the historical averaged block rewards
*/
public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(reward) as INT) as avg_rewards
FROM blocks`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot generate block rewards history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}

100
backend/src/repositories/HashratesRepository.ts
View File

@@ -1,5 +1,5 @@
import { Common } from '../api/common';
import { DB } from '../database';
import DB from '../database';
import logger from '../logger';
import PoolsRepository from './PoolsRepository';
@@ -20,14 +20,10 @@ class HashratesRepository {
}
query = query.slice(0, -1);
let connection;
try {
connection = await DB.getConnection();
await connection.query(query);
connection.release();
await DB.query(query);
} catch (e: any) {
connection.release();
logger.err('$saveHashrateInDatabase() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -35,8 +31,6 @@ class HashratesRepository {
public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
interval = Common.getSqlInterval(interval);
const connection = await DB.getConnection();
let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
FROM hashrates`;
@@ -50,33 +44,25 @@ class HashratesRepository {
query += ` ORDER by hashrate_timestamp`;
try {
const [rows]: any[] = await connection.query(query);
connection.release();
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
connection.release();
logger.err('$getNetworkDailyHashrate() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
const connection = await DB.getConnection();
const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp
FROM hashrates
WHERE type = 'weekly'
GROUP BY hashrate_timestamp`;
try {
const [rows]: any[] = await connection.query(query);
connection.release();
const [rows]: any[] = await DB.query(query);
return rows.map(row => row.timestamp);
} catch (e) {
connection.release();
logger.err('$getWeeklyHashrateTimestamps() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -87,7 +73,6 @@ class HashratesRepository {
public async $getPoolsWeeklyHashrate(interval: string | null): Promise<any[]> {
interval = Common.getSqlInterval(interval);
const connection = await DB.getConnection();
const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
@@ -106,13 +91,10 @@ class HashratesRepository {
query += ` ORDER by hashrate_timestamp, FIELD(pool_id, ${topPoolsId})`;
try {
const [rows]: any[] = await connection.query(query);
connection.release();
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
connection.release();
logger.err('$getPoolsWeeklyHashrate() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -120,13 +102,16 @@ class HashratesRepository {
/**
* Returns a pool hashrate history
*/
public async $getPoolWeeklyHashrate(poolId: number): Promise<any[]> {
const connection = await DB.getConnection();
public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error(`This mining pool does not exist`);
}
// Find hashrate boundaries
let query = `SELECT MIN(hashrate_timestamp) as firstTimestamp, MAX(hashrate_timestamp) as lastTimestamp
FROM hashrates
JOIN pools on pools.id = pool_id
FROM hashrates
JOIN pools on pools.id = pool_id
WHERE hashrates.type = 'weekly' AND pool_id = ? AND avg_hashrate != 0
ORDER by hashrate_timestamp LIMIT 1`;
@@ -134,13 +119,12 @@ class HashratesRepository {
firstTimestamp: '1970-01-01',
lastTimestamp: '9999-01-01'
};
try {
const [rows]: any[] = await connection.query(query, [poolId]);
const [rows]: any[] = await DB.query(query, [pool.id]);
boundaries = rows[0];
connection.release();
} catch (e) {
connection.release();
logger.err('$getPoolWeeklyHashrate() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
}
// Get hashrates entries between boundaries
@@ -152,47 +136,65 @@ class HashratesRepository {
ORDER by hashrate_timestamp`;
try {
const [rows]: any[] = await connection.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, poolId]);
connection.release();
const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
return rows;
} catch (e) {
connection.release();
logger.err('$getPoolWeeklyHashrate() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Set latest run timestamp
*/
public async $setLatestRunTimestamp(key: string, val: any = null) {
const connection = await DB.getConnection();
const query = `UPDATE state SET number = ? WHERE name = ?`;
try {
await connection.query<any>(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]);
connection.release();
await DB.query(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]);
} catch (e) {
connection.release();
logger.err(`Cannot set last indexing timestamp for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get latest run timestamp
*/
public async $getLatestRunTimestamp(key: string): Promise<number> {
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = ?`;
try {
const [rows] = await connection.query<any>(query, [key]);
connection.release();
const [rows]: any[] = await DB.query(query, [key]);
if (rows.length === 0) {
return 0;
}
return rows[0]['number'];
} catch (e) {
connection.release();
logger.err('$setLatestRunTimestamp() error' + (e instanceof Error ? e.message : e));
logger.err(`Cannot retreive last indexing timestamp for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Delete most recent data points for re-indexing
*/
public async $deleteLastEntries() {
logger.info(`Delete latest hashrates data points from the database`);
try {
const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
for (const row of rows) {
await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
}
// Re-run the hashrate indexing to fill up missing data
await this.$setLatestRunTimestamp('last_hashrates_indexing', 0);
await this.$setLatestRunTimestamp('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
}
}
}
export default new HashratesRepository();

53
backend/src/repositories/PoolsRepository.ts
View File

@@ -1,5 +1,6 @@
import { Common } from '../api/common';
import { DB } from '../database';
import config from '../config';
import DB from '../database';
import logger from '../logger';
import { PoolInfo, PoolTag } from '../mempool.interfaces';
@@ -8,9 +9,7 @@ class PoolsRepository {
* Get all pools tagging info
*/
public async $getPools(): Promise<PoolTag[]> {
const connection = await DB.getConnection();
const [rows] = await connection.query('SELECT id, name, addresses, regexes FROM pools;');
connection.release();
const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
return <PoolTag[]>rows;
}
@@ -18,9 +17,7 @@ class PoolsRepository {
* Get unknown pool tagging info
*/
public async $getUnknownPool(): Promise<PoolTag> {
const connection = await DB.getConnection();
const [rows] = await connection.query('SELECT id, name FROM pools where name = "Unknown"');
connection.release();
const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
return <PoolTag>rows[0];
}
@@ -30,7 +27,7 @@ class PoolsRepository {
public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
interval = Common.getSqlInterval(interval);
let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link
let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
FROM blocks
JOIN pools on pools.id = pool_id`;
@@ -41,16 +38,11 @@ class PoolsRepository {
query += ` GROUP BY pool_id
ORDER BY COUNT(height) DESC`;
// logger.debug(query);
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query);
connection.release();
const [rows] = await DB.query(query);
return <PoolInfo[]>rows;
} catch (e) {
connection.release();
logger.err('$getPoolsInfo() error' + (e instanceof Error ? e.message : e));
logger.err(`Cannot generate pools stats. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -64,15 +56,11 @@ class PoolsRepository {
LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
GROUP BY pools.id`;
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, [from, to]);
connection.release();
const [rows] = await DB.query(query, [from, to]);
return <PoolInfo[]>rows;
} catch (e) {
connection.release();
logger.err('$getPoolsInfoBetween() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot generate pools blocks count. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -80,25 +68,30 @@ class PoolsRepository {
/**
* Get mining pool statistics for one pool
*/
public async $getPool(poolId: any): Promise<object> {
public async $getPool(slug: string): Promise<PoolTag | null> {
const query = `
SELECT *
FROM pools
WHERE pools.id = ?`;
WHERE pools.slug = ?`;
// logger.debug(query);
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, [poolId]);
connection.release();
const [rows]: any[] = await DB.query(query, [slug]);
if (rows.length < 1) {
logger.debug(`This slug does not match any known pool`);
return null;
}
rows[0].regexes = JSON.parse(rows[0].regexes);
rows[0].addresses = JSON.parse(rows[0].addresses);
if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
rows[0].addresses = []; // pools.json only contains mainnet addresses
} else {
rows[0].addresses = JSON.parse(rows[0].addresses);
}
return rows[0];
} catch (e) {
connection.release();
logger.err('$getPool() error' + (e instanceof Error ? e.message : e));
logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}

68
backend/src/routes.ts
View File

@@ -539,20 +539,24 @@ class Routes {
public async $getPool(req: Request, res: Response) {
try {
const stats = await mining.$getPoolStat(parseInt(req.params.poolId, 10));
const stats = await mining.$getPoolStat(req.params.slug);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(stats);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
res.status(404).send(e.message);
} else {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
public async $getPoolBlocks(req: Request, res: Response) {
try {
const poolBlocks = await BlocksRepository.$getBlocksByPool(
parseInt(req.params.poolId, 10),
req.params.slug,
req.params.height === undefined ? undefined : parseInt(req.params.height, 10),
);
res.header('Pragma', 'public');
@@ -560,7 +564,11 @@ class Routes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(poolBlocks);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
res.status(404).send(e.message);
} else {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
@@ -576,18 +584,6 @@ class Routes {
}
}
public async $getHistoricalDifficulty(req: Request, res: Response) {
try {
const stats = await BlocksRepository.$getBlocksDifficulty(req.params.interval ?? null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json(stats);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async $getPoolsHistoricalHashrate(req: Request, res: Response) {
try {
const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval ?? null);
@@ -606,7 +602,7 @@ class Routes {
public async $getPoolHistoricalHashrate(req: Request, res: Response) {
try {
const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(parseInt(req.params.poolId, 10));
const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
@@ -616,7 +612,11 @@ class Routes {
hashrates: hashrates,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
res.status(404).send(e.message);
} else {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
@@ -638,6 +638,38 @@ class Routes {
}
}
public async $getHistoricalBlockFees(req: Request, res: Response) {
try {
const blockFees = await mining.$getHistoricalBlockFees(req.params.interval ?? null);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
blockFees: blockFees,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async $getHistoricalBlockRewards(req: Request, res: Response) {
try {
const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval ?? null);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
blockRewards: blockRewards,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async getBlock(req: Request, res: Response) {
try {
const result = await bitcoinApi.$getBlock(req.params.hash);

140
backend/src/tasks/pools-updater.ts Normal file
View File

@@ -0,0 +1,140 @@
const https = require('https');
import poolsParser from '../api/pools-parser';
import config from '../config';
import DB from '../database';
import logger from '../logger';
/**
* Maintain the most recent version of pools.json
*/
class PoolsUpdater {
lastRun: number = 0;
constructor() {
}
public async updatePoolsJson() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
const oneWeek = 604800;
const oneDay = 86400;
const now = new Date().getTime() / 1000;
if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart
return;
}
this.lastRun = now;
try {
const dbSha = await this.getShaFromDb();
const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
if (githubSha === undefined) {
return;
}
logger.debug(`Pools.json sha | Current: ${dbSha} | Github: ${githubSha}`);
if (dbSha !== undefined && dbSha === githubSha) {
return;
}
logger.warn('Pools.json is outdated, fetch latest from github');
const poolsJson = await this.fetchPools();
await poolsParser.migratePoolsJson(poolsJson);
await this.updateDBSha(githubSha);
logger.notice('PoolsUpdater completed');
} catch (e) {
this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Fetch pools.json from github repo
*/
private async fetchPools(): Promise<object> {
const response = await this.query('/repos/mempool/mining-pools/contents/pools.json');
return JSON.parse(Buffer.from(response['content'], 'base64').toString('utf8'));
}
/**
* Fetch our latest pools.json sha from the db
*/
private async updateDBSha(githubSha: string) {
try {
await DB.query('DELETE FROM state where name="pools_json_sha"');
await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
} catch (e) {
logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
return undefined;
}
}
/**
* Fetch our latest pools.json sha from the db
*/
private async getShaFromDb(): Promise<string | undefined> {
try {
const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
return (rows.length > 0 ? rows[0].string : undefined);
} catch (e) {
logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
return undefined;
}
}
/**
* Fetch our latest pools.json sha from github
*/
private async fetchPoolsSha(): Promise<string | undefined> {
const response = await this.query('/repos/mempool/mining-pools/git/trees/master');
for (const file of response['tree']) {
if (file['path'] === 'pools.json') {
return file['sha'];
}
}
logger.err('Cannot to find latest pools.json sha from github api response');
return undefined;
}
/**
* Http request wrapper
*/
private query(path): Promise<string> {
return new Promise((resolve, reject) => {
const options = {
host: 'api.github.com',
path: path,
method: 'GET',
headers: { 'user-agent': 'node.js' }
};
logger.debug('Querying: api.github.com' + path);
const request = https.get(options, (response) => {
const chunks_of_data: any[] = [];
response.on('data', (fragments) => {
chunks_of_data.push(fragments);
});
response.on('end', () => {
resolve(JSON.parse(Buffer.concat(chunks_of_data).toString()));
});
response.on('error', (error) => {
reject(error);
});
});
request.on('error', (error) => {
logger.err('Github API query failed. Reason: ' + error);
reject(error);
});
});
}
}
export default new PoolsUpdater();

1
backend/src/utils/blocks-utils.ts
View File

@@ -23,6 +23,7 @@ export function prepareBlock(block: any): BlockExtended {
pool: block?.extras?.pool ?? (block?.pool_id ? {
id: block.pool_id,
name: block.pool_name,
slug: block.pool_slug,
} : undefined),
}
};