machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Index difficulty adjustments

Browse Source
This commit is contained in:
nymkappa
2022-06-25 12:14:32 +02:00
parent 570d8cfc74
commit acfdc8163b
14 changed files with 225 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend/src/api/blocks.ts
View File

@@ -19,6 +19,9 @@ import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import difficultyAdjustment from './difficulty-adjustment';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -449,7 +452,10 @@ class Blocks {
const newBlock = await this.$indexBlock(lastBlock['height'] - i);
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
}
logger.info(`Re-indexed 10 blocks and summaries`);
await mining.$indexDifficultyAdjustments();
await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
indexer.reindex();
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
@@ -461,6 +467,15 @@ class Blocks {
}
if (block.height % 2016 === 0) {
if (Common.indexingEnabled()) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: block.timestamp,
height: block.height,
difficulty: block.difficulty,
adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
});
}
this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty;

20
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 21;
private static currentVersion = 22;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -226,6 +226,11 @@ class DatabaseMigration {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
}
if (databaseSchemaVersion < 22 && isBitcoin === true) {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
}
} catch (e) {
throw e;
}
@@ -513,7 +518,7 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateRatesTableQuery(): string {
private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table
return `CREATE TABLE IF NOT EXISTS rates (
height int(10) unsigned NOT NULL,
bisq_rates JSON NOT NULL,
@@ -539,6 +544,17 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateDifficultyAdjustmentsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS difficulty_adjustments (
time timestamp NOT NULL,
height int(10) unsigned NOT NULL,
difficulty double unsigned NOT NULL,
adjustment float NOT NULL,
PRIMARY KEY (height),
INDEX (time)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];

45
backend/src/api/mining.ts
View File

@@ -1,4 +1,4 @@
import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
import BlocksRepository from '../repositories/BlocksRepository';
import PoolsRepository from '../repositories/PoolsRepository';
import HashratesRepository from '../repositories/HashratesRepository';
@@ -7,6 +7,7 @@ import logger from '../logger';
import { Common } from './common';
import loadingIndicators from './loading-indicators';
import { escape } from 'mysql2';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
class Mining {
constructor() {
@@ -377,6 +378,48 @@ class Mining {
}
}
/**
* Index difficulty adjustments
*/
public async $indexDifficultyAdjustments(): Promise<void> {
const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
const indexedHeights = {};
for (const height of indexedHeightsArray) {
indexedHeights[height] = true;
}
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
let currentDifficulty = 0;
let totalIndexed = 0;
for (const block of blocks) {
if (block.difficulty !== currentDifficulty) {
if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed
currentDifficulty = block.difficulty;
continue;
}
let adjustment = block.difficulty / Math.max(1, currentDifficulty);
adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: block.time,
height: block.height,
difficulty: block.difficulty,
adjustment: adjustment,
});
totalIndexed++;
currentDifficulty = block.difficulty;
}
}
if (totalIndexed > 0) {
logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
}
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);

1
backend/src/index.ts
View File

@@ -290,6 +290,7 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments)
;
}

1
backend/src/indexer.ts
View File

@@ -36,6 +36,7 @@ class Indexer {
return;
}
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();

7
backend/src/mempool.interfaces.ts
View File

@@ -224,6 +224,13 @@ export interface IDifficultyAdjustment {
timeOffset: number;
}
export interface IndexedDifficultyAdjustment {
time: number; // UNIX timestamp
height: number; // Block height
difficulty: number;
adjustment: number;
}
export interface RewardStats {
totalReward: number;
totalFee: number;

65
backend/src/repositories/BlocksRepository.ts
View File

@@ -7,6 +7,7 @@ import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2';
import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
class BlocksRepository {
/**
@@ -381,48 +382,9 @@ class BlocksRepository {
/**
* Return blocks difficulty
*/
public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
interval = Common.getSqlInterval(interval);
// :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
// Basically, using temporary user defined fields, we are able to extract all
// difficulty adjustments from the blocks tables.
// This allow use to avoid indexing it in another table.
let query = `
SELECT
*
FROM
(
SELECT
UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height,
IF(@prevStatus = YT.difficulty, @rn := @rn + 1,
IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1)
) AS rn
FROM blocks YT
CROSS JOIN
(
SELECT @prevStatus := -1, @rn := 1
) AS var
`;
if (interval) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += `
ORDER BY YT.height
) AS t
WHERE t.rn = 1
ORDER BY t.height
`;
public async $getBlocksDifficulty(): Promise<object[]> {
try {
const [rows]: any[] = await DB.query(query);
for (const row of rows) {
delete row['rn'];
}
const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
return rows;
} catch (e) {
logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
@@ -452,26 +414,6 @@ class BlocksRepository {
}
}
/*
* Check if the last 10 blocks chain is valid
*/
public async $validateRecentBlocks(): Promise<boolean> {
try {
const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
for (let i = 0; i < lastBlocks.length - 1; ++i) {
if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
return false;
}
}
return true;
} catch (e) {
return true; // Don't do anything if there is a db error
}
}
/**
* Check if the chain of block hash is valid and delete data from the stale branch if needed
*/
@@ -498,6 +440,7 @@ class BlocksRepository {
await this.$deleteBlocksFrom(blocks[idx - 1].height);
await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
return false;
}
++idx;

88
backend/src/repositories/DifficultyAdjustmentsRepository.ts Normal file
View File

@@ -0,0 +1,88 @@
import { Common } from '../api/common';
import DB from '../database';
import logger from '../logger';
import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
class DifficultyAdjustmentsRepository {
public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise<void> {
if (adjustment.height === 1) {
return;
}
try {
const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`;
const params: any[] = [
adjustment.time,
adjustment.height,
adjustment.difficulty,
adjustment.adjustment,
];
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
} else {
logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
}
public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
interval = Common.getSqlInterval(interval);
let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment
FROM difficulty_adjustments`;
if (interval) {
query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
if (descOrder === true) {
query += ` ORDER BY time DESC`;
} else {
query += ` ORDER BY time`;
}
try {
const [rows] = await DB.query(query);
return rows as IndexedDifficultyAdjustment[];
} catch (e) {
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getAdjustmentsHeights(): Promise<number[]> {
try {
const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
return rows.map(block => block.height);
} catch (e: any) {
logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
try {
logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
} catch (e: any) {
logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $deleteLastAdjustment(): Promise<void> {
try {
logger.info(`Delete last difficulty adjustment from the database`);
await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
} catch (e: any) {
logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
}
export default new DifficultyAdjustmentsRepository();

15
backend/src/routes.ts
View File

@@ -26,6 +26,7 @@ import mining from './api/mining';
import BlocksRepository from './repositories/BlocksRepository';
import HashratesRepository from './repositories/HashratesRepository';
import difficultyAdjustment from './api/difficulty-adjustment';
import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository';
class Routes {
constructor() {}
@@ -653,7 +654,7 @@ class Routes {
try {
const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
@@ -730,6 +731,18 @@ class Routes {
}
}
public async $getDifficultyAdjustments(req: Request, res: Response) {
try {
const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, true);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async getBlock(req: Request, res: Response) {
try {
const block = await blocks.$getBlock(req.params.hash);