Basic block indexing WIP - Default mining pool icon - Only show mining hashrate on 1d scale

backend/src/api/blocks.ts

@@ -7,10 +7,10 @@ import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
 import bitcoinClient from './bitcoin/bitcoin-client';
-import { DB } from '../database';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
+import BitcoinApi from './bitcoin/bitcoin-api';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -146,22 +146,41 @@ class Blocks {
    * Index all blocks metadata for the mining dashboard
    */
   public async $generateBlockDatabase() {
-    let currentBlockHeight = await bitcoinApi.$getBlockHeightTip();
-    let maxBlocks = 1008*2; // tmp
+    let currentBlockHeight = await bitcoinClient.getBlockCount();
+    const indexedBlockCount = await blocksRepository.$blockCount();
 
-    while (currentBlockHeight-- > 0 && maxBlocks-- > 0) {
-      if (await blocksRepository.$isBlockAlreadyIndexed(currentBlockHeight)) {
-        // logger.debug(`Block #${currentBlockHeight} already indexed, skipping`);
+    logger.info(`Starting block indexing. Current tip at block #${currentBlockHeight}`);
+    logger.info(`Need to index ${currentBlockHeight - indexedBlockCount} blocks. Working on it!`);
+
+    const chunkSize = 10000;
+    while (currentBlockHeight >= 0) {
+      const endBlock = Math.max(0, currentBlockHeight - chunkSize + 1);
+      const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
+        currentBlockHeight, endBlock);
+      if (missingBlockHeights.length <= 0) {
+        logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}, moving on`);
+        currentBlockHeight -= chunkSize;
         continue;
       }
-      logger.debug(`Indexing block #${currentBlockHeight}`);
-      const blockHash = await bitcoinApi.$getBlockHash(currentBlockHeight);
-      const block = await bitcoinApi.$getBlock(blockHash);
-      const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
-      const blockExtended = this.getBlockExtended(block, transactions);
-      const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
-      const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
-      await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
+
+      logger.info(`Indexing ${chunkSize} blocks from #${currentBlockHeight} to #${endBlock}`);
+
+      for (const blockHeight of missingBlockHeights) {
+        try {
+          logger.debug(`Indexing block #${blockHeight}`);
+          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
+          const block = await bitcoinApi.$getBlock(blockHash);
+          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+          const blockExtended = this.getBlockExtended(block, transactions);
+          const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
+          const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
+          await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
+        } catch (e) {
+          logger.err(`Something went wrong while indexing blocks.` + e);
+        }
+      }
+
+      currentBlockHeight -= chunkSize;
     }
   }
 

backend/src/api/database-migration.ts

@@ -6,7 +6,7 @@ import logger from '../logger';
 const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
 
 class DatabaseMigration {
-  private static currentVersion = 3;
+  private static currentVersion = 4;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
 
@@ -85,6 +85,8 @@ class DatabaseMigration {
       }
       if (databaseSchemaVersion < 3) {
         await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+      }
+      if (databaseSchemaVersion < 4) {
         await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
       }
       connection.release();

backend/src/api/mining.ts

@@ -1,33 +1,30 @@
-import { PoolInfo, PoolStats } from "../mempool.interfaces";
-import BlocksRepository, { EmptyBlocks } from "../repositories/BlocksRepository";
-import PoolsRepository from "../repositories/PoolsRepository";
-import bitcoinClient from "./bitcoin/bitcoin-client";
-import BitcoinApi from "./bitcoin/bitcoin-api";
+import { PoolInfo, PoolStats } from '../mempool.interfaces';
+import BlocksRepository, { EmptyBlocks } from '../repositories/BlocksRepository';
+import PoolsRepository from '../repositories/PoolsRepository';
+import bitcoinClient from './bitcoin/bitcoin-client';
+import BitcoinApi from './bitcoin/bitcoin-api';
 
 class Mining {
-  private bitcoinApi: BitcoinApi;
-
   constructor() {
-    this.bitcoinApi = new BitcoinApi(bitcoinClient);
   }
 
   /**
    * Generate high level overview of the pool ranks and general stats
    */
-  public async $getPoolsStats(interval: string = "100 YEAR") : Promise<object> {
-    let poolsStatistics = {};
+  public async $getPoolsStats(interval: string = '100 YEAR') : Promise<object> {
+    const poolsStatistics = {};
 
-    const blockHeightTip = await this.bitcoinApi.$getBlockHeightTip();
-    const lastBlockHashrate = await this.bitcoinApi.$getEstimatedHashrate(blockHeightTip);
+    const blockHeightTip = await bitcoinClient.getBlockCount();
+    const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(120, blockHeightTip);
     const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
     const blockCount: number = await BlocksRepository.$blockCount(interval);
     const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(interval);
 
-    let poolsStats: PoolStats[] = [];
+    const poolsStats: PoolStats[] = [];
     let rank = 1;
 
     poolsInfo.forEach((poolInfo: PoolInfo) => {
-      let poolStat: PoolStats = {
+      const poolStat: PoolStats = {
         poolId: poolInfo.poolId, // mysql row id
         name: poolInfo.name,
         link: poolInfo.link,
@@ -41,11 +38,11 @@ class Mining {
         }
       }
       poolsStats.push(poolStat);
-    })
+    });
 
-    poolsStatistics["blockCount"] = blockCount;
-    poolsStatistics["lastEstimatedHashrate"] = lastBlockHashrate;
-    poolsStatistics["pools"] = poolsStats;
+    poolsStatistics['blockCount'] = blockCount;
+    poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
+    poolsStatistics['pools'] = poolsStats;
 
     return poolsStatistics;
   }

backend/src/index.ts

@@ -26,6 +26,7 @@ import poolsParser from './api/pools-parser';
 import syncAssets from './sync-assets';
 import icons from './api/liquid/icons';
 import { Common } from './api/common';
+import bitcoinClient from './api/bitcoin/bitcoin-client';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -139,10 +140,13 @@ class Server {
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
 
-      if (this.blockIndexingStarted === false/* && memPool.isInSync()*/) {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      if (this.blockIndexingStarted === false
+        && memPool.isInSync()
+        && blockchainInfo.blocks === blockchainInfo.headers
+      ) {
         blocks.$generateBlockDatabase();
         this.blockIndexingStarted = true;
-        logger.info("START OLDER BLOCK INDEXING");
       }
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);

backend/src/repositories/BlocksRepository.ts

@@ -1,10 +1,10 @@
-import { BlockExtended, PoolTag } from "../mempool.interfaces";
-import { DB } from "../database";
-import logger from "../logger";
+import { BlockExtended, PoolTag } from '../mempool.interfaces';
+import { DB } from '../database';
+import logger from '../logger';
 
 export interface EmptyBlocks {
-  emptyBlocks: number,
-  poolId: number,
+  emptyBlocks: number;
+  poolId: number;
 }
 
 class BlocksRepository {
@@ -21,9 +21,9 @@ class BlocksRepository {
 
     try {
       const query = `INSERT INTO blocks(
-        height,  hash,     timestamp,    size,
-        weight,  tx_count, coinbase_raw, difficulty,
-        pool_id, fees,     fee_span,     median_fee
+        height,  hash,     blockTimestamp, size,
+        weight,  tx_count, coinbase_raw,   difficulty,
+        pool_id, fees,     fee_span,       median_fee
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
@@ -32,8 +32,8 @@ class BlocksRepository {
 
       const params: any[] = [
         block.height, blockHash, block.timestamp, block.size,
-        block.weight, block.tx_count, coinbaseHex ? coinbaseHex : "", block.difficulty,
-        poolTag.id, 0, "[]", block.medianFee,
+        block.weight, block.tx_count, coinbaseHex ? coinbaseHex : '', block.difficulty,
+        poolTag.id, 0, '[]', block.medianFee,
       ];
 
       await connection.query(query, params);
@@ -61,15 +61,36 @@ class BlocksRepository {
     return exists;
   }
 
+  /**
+   * Get all block height that have not been indexed between [startHeight, endHeight]
+   */
+  public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
+    const connection = await DB.pool.getConnection();
+    const [rows] : any[] = await connection.query(`
+      SELECT height
+      FROM blocks
+      WHERE height <= ${startHeight} AND height >= ${endHeight}
+      ORDER BY height DESC;
+    `);
+    connection.release();
+
+    const indexedBlockHeights: number[] = [];
+    rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
+    const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
+    const missingBlocksHeights =  seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
+
+    return missingBlocksHeights;
+  }
+
   /**
    * Count empty blocks for all pools
    */
-  public async $countEmptyBlocks(interval: string = "100 YEAR") : Promise<EmptyBlocks[]> {
+  public async $countEmptyBlocks(interval: string = '100 YEAR'): Promise<EmptyBlocks[]> {
     const connection = await DB.pool.getConnection();
     const [rows] = await connection.query(`
       SELECT pool_id as poolId
       FROM blocks
-      WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+      WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
       AND tx_count = 1;
     `);
     connection.release();
@@ -80,12 +101,12 @@ class BlocksRepository {
   /**
    * Get blocks count for a period
    */
-   public async $blockCount(interval: string = "100 YEAR") : Promise<number> {
+   public async $blockCount(interval: string = '100 YEAR'): Promise<number> {
     const connection = await DB.pool.getConnection();
     const [rows] = await connection.query(`
       SELECT count(height) as blockCount
       FROM blocks
-      WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW();
+      WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW();
     `);
     connection.release();
 

backend/src/repositories/PoolsRepository.ts

@@ -31,7 +31,7 @@ class PoolsRepository {
       SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link
       FROM blocks
       JOIN pools on pools.id = pool_id
-      WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+      WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
       GROUP BY pool_id
       ORDER BY COUNT(height) DESC;
     `);