fixes for non-dual-node rbf feature

blockchain momentum scrolling

.github/CODEOWNERS

@@ -0,0 +1 @@
+backend/src/api/database-migration.ts	@wiz @softsimon

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.16.0", "18.5.0"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"
@@ -55,7 +55,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.15.0", "18.5.0"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"

.github/workflows/cypress.yml

@@ -1,8 +1,11 @@
 name: Cypress Tests
 
 on:
+  push:
+    branches: [master]
   pull_request:
-    types: [opened, review_requested, synchronize]
+    types: [opened, synchronize]
+
 jobs:
   cypress:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"

.vscode/settings.json

@@ -1,4 +1,5 @@
 {
   "editor.tabSize": 2,
+  "typescript.preferences.importModuleSpecifier": "relative",
   "typescript.tsdk": "./backend/node_modules/typescript/lib"
 }

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project
-Copyright (c) 2019-2022 The Mempool Open Source Project Developers
+Copyright (c) 2019-2023 The Mempool Open Source Project Developers
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of (at your option) either:

README.md

@@ -1,5 +1,7 @@
 # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
+https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4
+
 Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
 It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.

backend/README.md

@@ -160,7 +160,7 @@ npm install -g ts-node nodemon
 Then, run the watcher:
 
 ```
-nodemon src/index.ts --ignore cache/ --ignore pools.json
+nodemon src/index.ts --ignore cache/
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -218,3 +218,31 @@ Generate block at regular interval (every 10 seconds in this example):
    ```
    watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
    ```
+
+### Mining pools update
+
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
+
+To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
+
+You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+
+When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+
+### Re-index tables
+
+You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
+
+Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
+
+Usage:
+```
+npm run start --reindex=blocks,hashrates
+```
+Example output:
+```
+Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
+Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
+```
+
+Reference: https://github.com/mempool/mempool/pull/1269

backend/mempool-config.sample.json

@@ -15,7 +15,6 @@
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "INDEXING_BLOCKS_AMOUNT": 11000,
     "BLOCKS_SUMMARIES_INDEXING": false,
-    "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": [],
     "EXTERNAL_MAX_RETRY": 1,
@@ -23,8 +22,9 @@
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false

backend/package.json

@@ -27,7 +27,7 @@
     "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
     "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
+    "start-production": "node --max-old-space-size=16384 dist/index.js",
     "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",

backend/src/__fixtures__/mempool-config.template.json

@@ -3,12 +3,11 @@
     "ENABLED": true,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
-    "ENABLED": true,
     "BLOCKS_SUMMARIES_INDEXING": true,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
-    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "AUTOMATIC_BLOCK_REINDEXING": false,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
     "CLEAR_PROTECTION_MINUTES": 4,
@@ -16,7 +15,6 @@
     "BLOCK_WEIGHT_UNITS": 6,
     "INITIAL_BLOCKS_AMOUNT": 7,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
-    "PRICE_FEED_UPDATE_INTERVAL": 9,
     "USE_SECOND_NODE_FOR_MINFEE": 10,
     "EXTERNAL_ASSETS": 11,
     "EXTERNAL_MAX_RETRY": 12,
@@ -26,9 +24,11 @@
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "AUDIT": "__MEMPOOL_AUDIT__",
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
-    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -29,7 +29,6 @@ describe('Mempool Backend Config', () => {
         INITIAL_BLOCKS_AMOUNT: 8,
         MEMPOOL_BLOCKS_AMOUNT: 8,
         INDEXING_BLOCKS_AMOUNT: 11000,
-        PRICE_FEED_UPDATE_INTERVAL: 600,
         USE_SECOND_NODE_FOR_MINFEE: false,
         EXTERNAL_ASSETS: [],
         EXTERNAL_MAX_RETRY: 1,
@@ -37,10 +36,12 @@ describe('Mempool Backend Config', () => {
         USER_AGENT: 'mempool',
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+        AUDIT: false,
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
         CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/audit.ts

@@ -119,7 +119,8 @@ class Audit {
     }
 
     const numCensored = Object.keys(isCensored).length;
-    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
 
     return {
       censored: Object.keys(isCensored),

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -172,4 +172,35 @@ export namespace IBitcoinApi {
     }
   }
 
+  export interface BlockStats {
+    "avgfee": number;
+    "avgfeerate": number;
+    "avgtxsize": number;
+    "blockhash": string;
+    "feerate_percentiles": [number, number, number, number, number];
+    "height": number;
+    "ins": number;
+    "maxfee": number;
+    "maxfeerate": number;
+    "maxtxsize": number;
+    "medianfee": number;
+    "mediantime": number;
+    "mediantxsize": number;
+    "minfee": number;
+    "minfeerate": number;
+    "mintxsize": number;
+    "outs": number;
+    "subsidy": number;
+    "swtotal_size": number;
+    "swtotal_weight": number;
+    "swtxs": number;
+    "time": number;
+    "total_out": number;
+    "total_size": number;
+    "total_weight": number;
+    "totalfee": number;
+    "txs": number;
+    "utxo_increase": number;
+    "utxo_size_inc": number;
+  }
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       size: block.size,
       weight: block.weight,
       previousblockhash: block.previousblockhash,
+      mediantime: block.mediantime,
     };
   }
 

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler';
 import mempool from '../mempool';
 import feeApi from '../fee-api';
 import mempoolBlocks from '../mempool-blocks';
-import bitcoinApi from './bitcoin-api-factory';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
 import { Common } from '../common';
 import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
@@ -32,8 +32,10 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
       .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/rbf', this.getRbfHistory)
       .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'replacements', this.getRbfReplacements)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fullrbf/replacements', this.getFullRbfReplacements)
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
@@ -95,6 +97,8 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -215,7 +219,15 @@ class BitcoinRoutes {
       res.json(cpfpInfo);
       return;
     } else {
-      const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      let cpfpInfo;
+      if (config.DATABASE.ENABLED) {
+        cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      } else {
+        res.json({
+          ancestors: []
+        });
+        return;
+      }
       if (cpfpInfo) {
         res.json(cpfpInfo);
         return;
@@ -402,6 +414,41 @@ class BitcoinRoutes {
     }
   }
 
+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!req.params.from || from < 0) {
+        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (to < 0) {
+        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+      }
+      if (from > to) {
+        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+      }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
+
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getLegacyBlocks(req: Request, res: Response) {
     try {
       const returnBlocks: IEsploraApi.Block[] = [];
@@ -424,7 +471,7 @@ class BitcoinRoutes {
           returnBlocks.push(localBlock);
           nextHash = localBlock.previousblockhash;
         } else {
-          const block = await bitcoinApi.$getBlock(nextHash);
+          const block = await bitcoinCoreApi.$getBlock(nextHash);
           returnBlocks.push(block);
           nextHash = block.previousblockhash;
         }
@@ -594,8 +641,30 @@ class BitcoinRoutes {
 
   private async getRbfHistory(req: Request, res: Response) {
     try {
-      const result = rbfCache.getReplaces(req.params.txId);
-      res.json(result || []);
+      const replacements = rbfCache.getRbfTree(req.params.txId) || null;
+      const replaces = rbfCache.getReplaces(req.params.txId) || null;
+      res.json({
+        replacements,
+        replaces
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRbfReplacements(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getRbfTrees(false);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getFullRbfReplacements(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getRbfTrees(true);
+      res.json(result);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -88,6 +88,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    mediantime: number;
   }
 
   export interface Address {

backend/src/api/bitcoin/esplora-api.ts

@@ -1,8 +1,13 @@
 import config from '../../config';
 import axios, { AxiosRequestConfig } from 'axios';
+import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 
+const axiosConnection = axios.create({
+  httpAgent: new http.Agent({ keepAlive: true })
+});
+
 class ElectrsApi implements AbstractBitcoinApi {
   axiosConfig: AxiosRequestConfig = {
     timeout: 10000,
@@ -11,52 +16,52 @@ class ElectrsApi implements AbstractBitcoinApi {
   constructor() { }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHashTip(): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -77,12 +82,12 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
       .then((response) => response.data);
   }
 

backend/src/api/blocks.ts

@@ -1,8 +1,8 @@
 import config from '../config';
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -13,11 +13,9 @@ import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
 import loadingIndicators from './loading-indicators';
 import BitcoinApi from './bitcoin/bitcoin-api';
-import { prepareBlock } from '../utils/blocks-utils';
 import BlocksRepository from '../repositories/BlocksRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
-import fiatConversion from './fiat-conversion';
 import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
@@ -26,6 +24,7 @@ import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
+import chainTips from './chain-tips';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -143,7 +142,7 @@ class Blocks {
    * @param block
    * @returns BlockSummary
    */
-  private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
+  public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
     const stripped = block.tx.map((tx) => {
       return {
         txid: tx.txid,
@@ -166,33 +165,81 @@ class Blocks {
    * @returns BlockExtended
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
-    blockExtended.extras.usd = fiatConversion.getConversionRates().USD;
+    const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    
+    const blk: Partial<BlockExtended> = Object.assign({}, block);
+    const extras: Partial<BlockExtension> = {};
+
+    extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
+    extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
 
     if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
+      extras.medianFee = 0; // 50th percentiles
+      extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      extras.totalFees = 0;
+      extras.avgFee = 0;
+      extras.avgFeeRate = 0;
+      extras.utxoSetChange = 0;
+      extras.avgTxSize = 0;
+      extras.totalInputs = 0;
+      extras.totalOutputs = 1;
+      extras.totalOutputAmt = 0;
+      extras.segwitTotalTxs = 0;
+      extras.segwitTotalSize = 0;
+      extras.segwitTotalWeight = 0;
     } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
-      ]);
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.totalFees = stats.totalfee;
-      blockExtended.extras.avgFee = stats.avgfee;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+      const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
+      extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      extras.totalFees = stats.totalfee;
+      extras.avgFee = stats.avgfee;
+      extras.avgFeeRate = stats.avgfeerate;
+      extras.utxoSetChange = stats.utxo_increase;
+      extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
+      extras.totalInputs = stats.ins;
+      extras.totalOutputs = stats.outs;
+      extras.totalOutputAmt = stats.total_out;
+      extras.segwitTotalTxs = stats.swtxs;
+      extras.segwitTotalSize = stats.swtotal_size;
+      extras.segwitTotalWeight = stats.swtotal_weight;
+    }
+
+    if (Common.blocksSummariesIndexingEnabled()) {
+      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+      if (extras.feePercentiles !== null) {
+        extras.medianFeeAmt = extras.feePercentiles[3];
+      }
+    }
+  
+    extras.virtualSize = block.weight / 4.0;
+    if (coinbaseTx?.vout.length > 0) {
+      extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+      extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
+    } else {
+      extras.coinbaseAddress = null;
+      extras.coinbaseSignature = null;
+      extras.coinbaseSignatureAscii = null;
+    }
+
+    const header = await bitcoinClient.getBlockHeader(block.id, false);
+    extras.header = header;
+
+    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
+    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+      extras.utxoSetSize = txoutset.txouts,
+      extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
+    } else {
+      extras.utxoSetSize = null;
+      extras.totalInputAmt = null;
     }
 
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      if (coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
           pool = await poolsRepository.$getUnknownPool();
@@ -202,23 +249,27 @@ class Blocks {
       }
 
       if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
       } else {
-        blockExtended.extras.pool = {
-          id: pool.id,
+        extras.pool = {
+          id: pool.uniqueId,
           name: pool.name,
           slug: pool.slug,
         };
       }
 
-      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
-      if (auditScore != null) {
-        blockExtended.extras.matchRate = auditScore.matchRate;
+      extras.matchRate = null;
+      if (config.MEMPOOL.AUDIT) {
+        const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
+        if (auditScore != null) {
+          extras.matchRate = auditScore.matchRate;
+        }
       }
     }
 
-    return blockExtended;
+    blk.extras = <BlockExtension>extras;
+    return <BlockExtended>blk;
   }
 
   /**
@@ -244,15 +295,18 @@ class Blocks {
     } else {
       pools = poolsParser.miningPools;
     }
+
     for (let i = 0; i < pools.length; ++i) {
       if (address !== undefined) {
-        const addresses: string[] = JSON.parse(pools[i].addresses);
+        const addresses: string[] = typeof pools[i].addresses === 'string' ?
+          JSON.parse(pools[i].addresses) : pools[i].addresses;
         if (addresses.indexOf(address) !== -1) {
           return pools[i];
         }
       }
 
-      const regexes: string[] = JSON.parse(pools[i].regexes);
+      const regexes: string[] = typeof pools[i].regexes === 'string' ?
+        JSON.parse(pools[i].regexes) : pools[i].regexes;
       for (let y = 0; y < regexes.length; ++y) {
         const regex = new RegExp(regexes[y], 'i');
         const match = asciiScriptSig.match(regex);
@@ -430,7 +484,7 @@ class Blocks {
             loadingIndicators.setProgress('block-indexing', progress, false);
           }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
-          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+          const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
           const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
           const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -478,13 +532,13 @@ class Blocks {
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
-        const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+        const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
+          const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -499,6 +553,7 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        await chainTips.updateOrphanedBlocks();
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -515,18 +570,18 @@ class Blocks {
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
-          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
-            logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
+          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
+            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
             // We assume there won't be a reorg with more than 10 block depth
-            await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
-            await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
+            await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
             for (let i = 10; i >= 0; --i) {
-              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
+              const newBlock = await this.$indexBlock(lastBlock.height - i);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
               if (config.MEMPOOL.CPFP_INDEXING) {
-                await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
+                await this.$indexCPFP(newBlock.id, lastBlock.height - i);
               }
             }
             await mining.$indexDifficultyAdjustments();
@@ -599,23 +654,27 @@ class Blocks {
    * Index a block if it's missing from the database. Returns the block after indexing
    */
   public async $indexBlock(height: number): Promise<BlockExtended> {
-    const dbBlock = await blocksRepository.$getBlockByHeight(height);
-    if (dbBlock != null) {
-      return prepareBlock(dbBlock);
+    if (Common.indexingEnabled()) {
+      const dbBlock = await blocksRepository.$getBlockByHeight(height);
+      if (dbBlock !== null) {
+        return dbBlock;
+      }
     }
 
     const blockHash = await bitcoinApi.$getBlockHash(height);
-    const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
     const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
 
-    await blocksRepository.$saveBlockInDatabase(blockExtended);
+    if (Common.indexingEnabled()) {
+      await blocksRepository.$saveBlockInDatabase(blockExtended);
+    }
 
-    return prepareBlock(blockExtended);
+    return blockExtended;
   }
 
   /**
-   * Index a block by hash if it's missing from the database. Returns the block after indexing
+   * Get one block by its hash
    */
   public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
     // Check the memory cache
@@ -624,31 +683,14 @@ class Blocks {
       return blockByHash;
     }
 
-    // Block has already been indexed
-    if (Common.indexingEnabled()) {
-      const dbBlock = await blocksRepository.$getBlockByHash(hash);
-      if (dbBlock != null) {
-        return prepareBlock(dbBlock);
-      }
-    }
-
-    // Not Bitcoin network, return the block as it
+    // Not Bitcoin network, return the block as it from the bitcoin backend
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return await bitcoinApi.$getBlock(hash);
+      return await bitcoinCoreApi.$getBlock(hash);
     }
 
-    let block = await bitcoinClient.getBlock(hash);
-    block = prepareBlock(block);
-
     // Bitcoin network, add our custom data on top
-    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
-    const blockExtended = await this.$getBlockExtended(block, transactions);
-    if (Common.indexingEnabled()) {
-      delete(blockExtended['coinbaseTx']);
-      await blocksRepository.$saveBlockInDatabase(blockExtended);
-    }
-
-    return blockExtended;
+    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
+    return await this.$indexBlock(block.height);
   }
 
   public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@@ -682,8 +724,19 @@ class Blocks {
     return summary.transactions;
   }
 
+  /**
+   * Get 15 blocks
+   * 
+   * Internally this function uses two methods to get the blocks, and
+   * the method is automatically selected:
+   *  - Using previous block hash links
+   *  - Using block height
+   * 
+   * @param fromHeight 
+   * @param limit 
+   * @returns 
+   */
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-
     let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
     if (currentHeight > this.currentBlockHeight) {
       limit -= currentHeight - this.currentBlockHeight;
@@ -695,27 +748,15 @@ class Blocks {
       return returnBlocks;
     }
 
-    // Check if block height exist in local cache to skip the hash lookup
-    const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
-    let startFromHash: string | null = null;
-    if (blockByHeight) {
-      startFromHash = blockByHeight.id;
-    } else if (!Common.indexingEnabled()) {
-      startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
-    }
-
-    let nextHash = startFromHash;
     for (let i = 0; i < limit && currentHeight >= 0; i++) {
       let block = this.getBlocks().find((b) => b.height === currentHeight);
       if (block) {
+        // Using the memory cache (find by height)
         returnBlocks.push(block);
-      } else if (Common.indexingEnabled()) {
+      } else {
+        // Using indexing (find by height, index on the fly, save in database)
         block = await this.$indexBlock(currentHeight);
         returnBlocks.push(block);
-      } else if (nextHash != null) {
-        block = prepareBlock(await bitcoinClient.getBlock(nextHash));
-        nextHash = block.previousblockhash;
-        returnBlocks.push(block);
       }
       currentHeight--;
     }
@@ -723,6 +764,114 @@ class Blocks {
     return returnBlocks;
   }
 
+  /**
+   * Used for bulk block data query
+   * 
+   * @param fromHeight 
+   * @param toHeight 
+   */
+  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
+    if (!Common.indexingEnabled()) {
+      return [];
+    }
+
+    const blocks: any[] = [];
+
+    while (fromHeight <= toHeight) {
+      let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
+      if (!block) {
+        await this.$indexBlock(fromHeight);
+        block = await blocksRepository.$getBlockByHeight(fromHeight);
+        if (!block) {
+          continue;
+        }
+      }
+
+      // Cleanup fields before sending the response
+      const cleanBlock: any = {
+        height: block.height ?? null,
+        hash: block.id ?? null,
+        timestamp: block.timestamp ?? null,
+        median_timestamp: block.mediantime ?? null,
+        previous_block_hash: block.previousblockhash ?? null,
+        difficulty: block.difficulty ?? null,
+        header: block.extras.header ?? null,
+        version: block.version ?? null,
+        bits: block.bits ?? null,
+        nonce: block.nonce ?? null,
+        size: block.size ?? null,
+        weight: block.weight ?? null,
+        tx_count: block.tx_count ?? null,
+        merkle_root: block.merkle_root ?? null,
+        reward: block.extras.reward ?? null,
+        total_fee_amt: block.extras.totalFees ?? null,
+        avg_fee_amt: block.extras.avgFee ?? null,
+        median_fee_amt: block.extras.medianFeeAmt ?? null,
+        fee_amt_percentiles: block.extras.feePercentiles ?? null,
+        avg_fee_rate: block.extras.avgFeeRate ?? null,
+        median_fee_rate: block.extras.medianFee ?? null,
+        fee_rate_percentiles: block.extras.feeRange ?? null,
+        total_inputs: block.extras.totalInputs ?? null,
+        total_input_amt: block.extras.totalInputAmt ?? null,
+        total_outputs: block.extras.totalOutputs ?? null,
+        total_output_amt: block.extras.totalOutputAmt ?? null,
+        segwit_total_txs: block.extras.segwitTotalTxs ?? null,
+        segwit_total_size: block.extras.segwitTotalSize ?? null,
+        segwit_total_weight: block.extras.segwitTotalWeight ?? null,
+        avg_tx_size: block.extras.avgTxSize ?? null,
+        utxoset_change: block.extras.utxoSetChange ?? null,
+        utxoset_size: block.extras.utxoSetSize ?? null,
+        coinbase_raw: block.extras.coinbaseRaw ?? null,
+        coinbase_address: block.extras.coinbaseAddress ?? null,
+        coinbase_signature: block.extras.coinbaseSignature ?? null,
+        coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
+        pool_slug: block.extras.pool.slug ?? null,
+        pool_id: block.extras.pool.id ?? null,
+      };
+
+      if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
+        cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        if (cleanBlock.fee_amt_percentiles === null) {
+          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
+          const summary = this.summarizeBlock(block);
+          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        }
+        if (cleanBlock.fee_amt_percentiles !== null) {
+          cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+        }
+      }
+
+      cleanBlock.fee_amt_percentiles = {
+        'min': cleanBlock.fee_amt_percentiles[0],
+        'perc_10': cleanBlock.fee_amt_percentiles[1],
+        'perc_25': cleanBlock.fee_amt_percentiles[2],
+        'perc_50': cleanBlock.fee_amt_percentiles[3],
+        'perc_75': cleanBlock.fee_amt_percentiles[4],
+        'perc_90': cleanBlock.fee_amt_percentiles[5],
+        'max': cleanBlock.fee_amt_percentiles[6],
+      };
+      cleanBlock.fee_rate_percentiles = {
+        'min': cleanBlock.fee_rate_percentiles[0],
+        'perc_10': cleanBlock.fee_rate_percentiles[1],
+        'perc_25': cleanBlock.fee_rate_percentiles[2],
+        'perc_50': cleanBlock.fee_rate_percentiles[3],
+        'perc_75': cleanBlock.fee_rate_percentiles[4],
+        'perc_90': cleanBlock.fee_rate_percentiles[5],
+        'max': cleanBlock.fee_rate_percentiles[6],
+      };
+
+      // Re-org can happen after indexing so we need to always get the
+      // latest state from core
+      cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
+
+      blocks.push(cleanBlock);
+      fromHeight++;
+    }
+
+    return blocks;
+  }
+
   public async $getBlockAuditSummary(hash: string): Promise<any> {
     let summary;
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {

backend/src/api/chain-tips.ts

@@ -0,0 +1,61 @@
+import logger from '../logger';
+import bitcoinClient from './bitcoin/bitcoin-client';
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: OrphanedBlock[] = [];
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+      this.orphanedBlocks = [];
+
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
+          let block = await bitcoinClient.getBlock(chain.hash);
+          while (block && block.confirmations === -1) {
+            this.orphanedBlocks.push({
+              height: block.height,
+              hash: block.hash,
+              status: chain.status
+            });
+            block = await bitcoinClient.getBlock(block.previousblockhash);
+          }
+        }
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
+    if (height === undefined) {
+      return [];
+    }
+
+    const orphans: OrphanedBlock[] = [];
+    for (const block of this.orphanedBlocks) {
+      if (block.height === height) {
+        orphans.push(block);
+      }
+    }
+    return orphans;
+  }
+}
+
+export default new ChainTips();

backend/src/api/common.ts

@@ -57,11 +57,11 @@ export class Common {
     return arr;
   }
 
-  static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } {
-    const matches: { [txid: string]: TransactionExtended } = {};
-    deleted
-      .forEach((deletedTx) => {
-        const foundMatches = added.find((addedTx) => {
+  static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended[] } {
+    const matches: { [txid: string]: TransactionExtended[] } = {};
+    added
+      .forEach((addedTx) => {
+        const foundMatches = deleted.filter((deletedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
           return addedTx.fee > deletedTx.fee
             // The new transaction must pay more fee per kB than the replaced tx.
@@ -70,8 +70,8 @@ export class Common {
             && deletedTx.vin.some((deletedVin) =>
               addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
-        if (foundMatches) {
-          matches[deletedTx.txid] = foundMatches;
+        if (foundMatches?.length) {
+          matches[addedTx.txid] = foundMatches;
         }
       });
     return matches;
@@ -237,14 +237,21 @@ export class Common {
     ].join('x');
   }
 
-  static utcDateToMysql(date?: number): string {
+  static utcDateToMysql(date?: number | null): string | null {
+    if (date === null) {
+      return null;
+    }
     const d = new Date((date || 0) * 1000);
     return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
   }
 
   static findSocketNetwork(addr: string): {network: string | null, url: string} {
     let network: string | null = null;
-    let url = addr.split('://')[1];
+    let url: string = addr;
+
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      url = addr.split('://')[1];
+    }
 
     if (!url) {
       return {
@@ -261,7 +268,7 @@ export class Common {
       }
     } else if (addr.indexOf('i2p') !== -1) {
       network = 'i2p';
-    } else if (addr.indexOf('ipv4') !== -1) {
+    } else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
       const ipv = isIP(url.split(':')[0]);
       if (ipv === 4) {
         network = 'ipv4';
@@ -271,7 +278,7 @@ export class Common {
           url: addr,
         };
       }
-    } else if (addr.indexOf('ipv6') !== -1) {
+    } else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
       url = url.split('[')[1].split(']')[0];
       const ipv = isIP(url);
       if (ipv === 6) {

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 52;
+  private static currentVersion = 57;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -62,8 +62,8 @@ class DatabaseMigration {
 
     if (databaseSchemaVersion <= 2) {
       // Disable some spam logs when they're not relevant
-      this.uniqueLogs.push(this.blocksTruncatedMessage);
-      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
     }
 
     logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -86,7 +86,7 @@ class DatabaseMigration {
       try {
         await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
         if (databaseSchemaVersion === 0) {
-          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
         } else {
           logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
         }
@@ -300,7 +300,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
       await this.updateToSchemaVersion(27);
     }
-    
+
     if (databaseSchemaVersion < 28 && isBitcoin === true) {
       if (config.LIGHTNING.ENABLED) {
         this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
@@ -464,10 +464,47 @@ class DatabaseMigration {
         await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
         await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
         await this.updateToSchemaVersion(52);
-      } catch(e) {
+      } catch (e) {
         logger.warn('' + (e instanceof Error ? e.message : e));
       }
     }
+
+    if (databaseSchemaVersion < 53) {
+      await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
+      await this.updateToSchemaVersion(53);
+    }
+
+    if (databaseSchemaVersion < 54) {
+      this.uniqueLog(logger.notice, `'prices' table has been truncated`);
+      await this.$executeQuery(`TRUNCATE prices`);
+      if (isBitcoin === true) {
+        this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
+        await this.$executeQuery(`TRUNCATE blocks_prices`);
+      }
+      await this.updateToSchemaVersion(54);
+    }
+
+    if (databaseSchemaVersion < 55) {
+      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+      await this.updateToSchemaVersion(55);
+    }
+
+    if (databaseSchemaVersion < 56) {
+      await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
+      await this.$executeQuery('TRUNCATE TABLE `blocks`');
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('DELETE FROM `pools`');
+      await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+      this.uniqueLog(logger.notice, '`pools` table has been truncated`');
+      await this.updateToSchemaVersion(56);
+    }
+
+    if (databaseSchemaVersion < 57 && isBitcoin === true) {
+      await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
+      await this.updateToSchemaVersion(57);
+    }
   }
 
   /**
@@ -591,7 +628,7 @@ class DatabaseMigration {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
     }
 
-    if (version < 9  && isBitcoin === true) {
+    if (version < 9 && isBitcoin === true) {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
     }
 
@@ -741,6 +778,28 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getAdditionalBlocksDataQuery(): string {
+    return `ALTER TABLE blocks
+      ADD median_timestamp timestamp NOT NULL,
+      ADD coinbase_address varchar(100) NULL,
+      ADD coinbase_signature varchar(500) NULL,
+      ADD coinbase_signature_ascii varchar(500) NULL,
+      ADD avg_tx_size double unsigned NOT NULL,
+      ADD total_inputs int unsigned NOT NULL,
+      ADD total_outputs int unsigned NOT NULL,
+      ADD total_output_amt bigint unsigned NOT NULL,
+      ADD fee_percentiles longtext NULL,
+      ADD median_fee_amt int unsigned NULL,
+      ADD segwit_total_txs int unsigned NOT NULL,
+      ADD segwit_total_size int unsigned NOT NULL,
+      ADD segwit_total_weight int unsigned NOT NULL,
+      ADD header varchar(160) NOT NULL,
+      ADD utxoset_change int NOT NULL,
+      ADD utxoset_size int unsigned NULL,
+      ADD total_input_amt bigint unsigned NULL
+    `;
+  }
+
   private getCreateDailyStatsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS hashrates (
       hashrate_timestamp timestamp NOT NULL,
@@ -958,26 +1017,17 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates', 'prices'];
+  public async $blocksReindexingTruncate(): Promise<void> {
+    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
+    await Common.sleep$(5000);
 
-    try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
-        }
-
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
-      }
-    } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
-    }
-  }
+    await this.$executeQuery(`TRUNCATE blocks`);
+    await this.$executeQuery(`TRUNCATE hashrates`);
+    await this.$executeQuery(`TRUNCATE difficulty_adjustments`);
+    await this.$executeQuery('DELETE FROM `pools`');
+    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+}
 
   private async $convertCompactCpfpTables(): Promise<void> {
     try {

backend/src/api/disk-cache.ts

@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
-  private cacheSchemaVersion = 1;
+  private cacheSchemaVersion = 3;
 
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
@@ -62,9 +62,24 @@ class DiskCache {
   }
 
   wipeCache() {
-    fs.unlinkSync(DiskCache.FILE_NAME);
+    logger.notice(`Wipping nodejs backend cache/cache*.json files`);
+    try {
+      fs.unlinkSync(DiskCache.FILE_NAME);
+    } catch (e: any) {
+      if (e?.code !== 'ENOENT') {
+        logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
+      }
+    }
+
     for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-      fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+      const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
+      try {
+        fs.unlinkSync(filename);
+      } catch (e: any) {
+        if (e?.code !== 'ENOENT') {
+          logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
+        }
+      }
     }
   }
 

backend/src/api/explorer/channels.api.ts

@@ -559,6 +559,17 @@ class ChannelsApi {
     const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
     const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
 
+    // https://github.com/mempool/mempool/issues/3006
+    if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
+      channel.last_update = null;
+    }
+    if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy1.last_update = null;
+    }
+    if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy2.last_update = null;
+    }
+
     const query = `INSERT INTO channels
       (
         id,

backend/src/api/explorer/nodes.api.ts

@@ -228,7 +228,7 @@ class NodesApi {
             nodes.capacity
           FROM nodes
           ORDER BY capacity DESC
-          LIMIT 100
+          LIMIT 6
         `;
 
         [rows] = await DB.query(query);
@@ -269,14 +269,26 @@ class NodesApi {
       let query: string;
       if (full === false) {
         query = `
-          SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
-            nodes.channels
+          SELECT
+            nodes.public_key as publicKey,
+            IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
           FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
           ORDER BY channels DESC
-          LIMIT 100;
+          LIMIT 6;
         `;
 
         [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
       } else {
         query = `
           SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
@@ -362,7 +374,13 @@ class NodesApi {
   public async $searchNodeByPublicKeyOrAlias(search: string) {
     try {
       const publicKeySearch = search.replace('%', '') + '%';
-      const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
+      const aliasSearch = search
+        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".  
+        .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
+        .split(' ')
+        .filter(key => key.length)
+        .map((search) => '+' + search + '*').join(' ');
+      // %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
       const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
       const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
       return rows;
@@ -624,6 +642,11 @@ class NodesApi {
    */
   public async $saveNode(node: ILightningApi.Node): Promise<void> {
     try {
+      // https://github.com/mempool/mempool/issues/3006
+      if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
+        node.last_update = null;
+      }
+  
       const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
       const query = `INSERT INTO nodes(
           public_key,

backend/src/api/fiat-conversion.ts

@@ -1,123 +0,0 @@
-import logger from '../logger';
-import * as http from 'http';
-import * as https from 'https';
-import axios, { AxiosResponse } from 'axios';
-import { IConversionRates } from '../mempool.interfaces';
-import config from '../config';
-import backendInfo from './backend-info';
-import { SocksProxyAgent } from 'socks-proxy-agent';
-
-class FiatConversion {
-  private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
-    'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
-    'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
-    'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
-  private conversionRates: IConversionRates = {};
-  private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
-  public ratesInitialized = false; // If true, it means rates are ready for use
-
-  constructor() {
-    for (const fiat of this.debasingFiatCurrencies) {
-      this.conversionRates[fiat] = 0;
-    }
-  }
-
-  public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
-    this.ratesChangedCallback = fn;
-  }
-
-  public startService() {
-    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
-    logger.info('Starting currency rates service');
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
-    } else {
-      logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
-    }
-    setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
-    this.updateCurrency();
-  }
-
-  public getConversionRates() {
-    return this.conversionRates;
-  }
-
-  private async updateCurrency(): Promise<void> {
-    type axiosOptions = {
-      headers: {
-        'User-Agent': string
-      };
-      timeout: number;
-      httpAgent?: http.Agent;
-      httpsAgent?: https.Agent;
-    }
-    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
-    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
-    const axiosOptions: axiosOptions = {
-      headers: {
-        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
-      },
-      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
-    };
-
-    let retry = 0;
-
-    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-      try {
-        if (config.SOCKS5PROXY.ENABLED) {
-          let socksOptions: any = {
-            agentOptions: {
-              keepAlive: true,
-            },
-            hostname: config.SOCKS5PROXY.HOST,
-            port: config.SOCKS5PROXY.PORT
-          };
-
-          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-            socksOptions.username = config.SOCKS5PROXY.USERNAME;
-            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-          } else {
-            // Retry with different tor circuits https://stackoverflow.com/a/64960234
-            socksOptions.username = `circuit${retry}`;
-          }
-
-          // Handle proxy agent for onion addresses
-          if (isHTTP) {
-            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
-          } else {
-            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-          }
-        }
-        
-        logger.debug('Querying currency rates service...');
-
-        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
-
-        if (response.statusText === 'error' || !response.data) {
-          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
-        }
-
-        for (const rate of response.data.data) {
-          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
-            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
-          }
-        }
-
-        this.ratesInitialized = true;
-        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
-
-        if (this.ratesChangedCallback) {
-          this.ratesChangedCallback(this.conversionRates);
-        }
-        break;
-      } catch (e) {
-        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
-        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
-        retry++;
-      }
-    }
-  }
-}
-
-export default new FiatConversion();

backend/src/api/lightning/lightning-api.interface.ts

@@ -21,7 +21,7 @@ export namespace ILightningApi {
   export interface Channel {
     channel_id: string;
     chan_point: string;
-    last_update: number;
+    last_update: number | null;
     node1_pub: string;
     node2_pub: string;
     capacity: string;
@@ -36,11 +36,11 @@ export namespace ILightningApi {
     fee_rate_milli_msat: string;
     disabled: boolean;
     max_htlc_msat: string;
-    last_update: number;
+    last_update: number | null;
   }
 
   export interface Node {
-    last_update: number;
+    last_update: number | null;
     pub_key: string;
     alias: string;
     addresses: {

backend/src/api/mempool-blocks.ts

@@ -97,14 +97,14 @@ class MempoolBlocks {
         blockSize += tx.size;
         transactions.push(tx);
       } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
         blockWeight = tx.weight;
         blockSize = tx.size;
         transactions = [tx];
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
     }
 
     return mempoolBlocks;
@@ -281,7 +281,7 @@ class MempoolBlocks {
     const mempoolBlocks = blocks.map((transactions, blockIndex) => {
       return this.dataToMempoolBlocks(transactions.map(tx => {
         return mempool[tx.txid] || null;
-      }).filter(tx => !!tx), undefined, undefined, blockIndex);
+      }).filter(tx => !!tx), blockIndex);
     });
 
     if (saveResults) {
@@ -293,18 +293,17 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[],
-    blockSize: number | undefined, blockWeight: number | undefined, blocksIndex: number): MempoolBlockWithTransactions {
-    let totalSize = blockSize || 0;
-    let totalWeight = blockWeight || 0;
-    if (blockSize === undefined && blockWeight === undefined) {
-      totalSize = 0;
-      totalWeight = 0;
-      transactions.forEach(tx => {
-        totalSize += tx.size;
-        totalWeight += tx.weight;
-      });
-    }
+  private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
+    let totalSize = 0;
+    let totalWeight = 0;
+    const fitTransactions: TransactionExtended[] = [];
+    transactions.forEach(tx => {
+      totalSize += tx.size;
+      totalWeight += tx.weight;
+      if ((totalWeight + tx.weight) <= config.MEMPOOL.BLOCK_WEIGHT_UNITS * 1.2) {
+        fitTransactions.push(tx);
+      }
+    });
     let rangeLength = 4;
     if (blocksIndex === 0) {
       rangeLength = 8;
@@ -322,7 +321,7 @@ class MempoolBlocks {
       medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
-      transactions: transactions.map((tx) => Common.stripTransaction(tx)),
+      transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
 }

backend/src/api/mempool.ts

@@ -206,13 +206,15 @@ class Mempool {
     logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
   }
 
-  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended[]; }): void {
     for (const rbfTransaction in rbfTransactions) {
-      if (this.mempoolCache[rbfTransaction]) {
+      if (this.mempoolCache[rbfTransaction] && rbfTransactions[rbfTransaction]?.length) {
         // Store replaced transactions
-        rbfCache.add(this.mempoolCache[rbfTransaction], rbfTransactions[rbfTransaction].txid);
+        rbfCache.add(rbfTransactions[rbfTransaction], this.mempoolCache[rbfTransaction]);
         // Erase the replaced transactions from the local mempool
-        delete this.mempoolCache[rbfTransaction];
+        for (const replaced of rbfTransactions[rbfTransaction]) {
+          delete this.mempoolCache[replaced.txid];
+        }
       }
     }
   }

backend/src/api/mining/mining-routes.ts

@@ -1,13 +1,13 @@
 import { Application, Request, Response } from 'express';
 import config from "../../config";
 import logger from '../../logger';
-import audits from '../audit';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
 import HashratesRepository from '../../repositories/HashratesRepository';
 import bitcoinClient from '../bitcoin/bitcoin-client';
 import mining from "./mining";
+import PricesRepository from '../../repositories/PricesRepository';
 
 class MiningRoutes {
   public initRoutes(app: Application) {
@@ -32,9 +32,27 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
     ;
   }
 
+  private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
+    try {
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      if (req.query.timestamp) {
+        res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
+          parseInt(<string>req.query.timestamp ?? 0, 10)
+        ));
+      } else {
+        res.status(200).send(await PricesRepository.$getHistoricalPrices());
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getPool(req: Request, res: Response): Promise<void> {
     try {
       const stats = await mining.$getPoolStat(req.params.slug);

backend/src/api/mining/mining.ts

@@ -11,6 +11,8 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
 import config from '../../config';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import PricesRepository from '../../repositories/PricesRepository';
+import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
+import { IEsploraApi } from '../bitcoin/esplora-api.interface';
 
 class Mining {
   blocksPriceIndexingRunning = false;
@@ -100,6 +102,7 @@ class Mining {
         rank: rank++,
         emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
         slug: poolInfo.slug,
+        avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
       };
       poolsStats.push(poolStat);
     });
@@ -171,7 +174,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate weekly mining pool hashrate history
+   * Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
@@ -188,8 +191,8 @@ class Mining {
     try {
       const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
-      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.time * 1000;
+      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
 
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
       const hashrates: any[] = [];
@@ -278,7 +281,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate daily hashrate data
+   * Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // We only run this once a day around midnight
@@ -291,8 +294,8 @@ class Mining {
     const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
     try {
-      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.time * 1000;
+      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
       const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       const lastMidnight = this.getDateMidnight(new Date());
       let toTimestamp = Math.round(lastMidnight.getTime());
@@ -393,13 +396,13 @@ class Mining {
     }
 
     const blocks: any = await BlocksRepository.$getBlocksDifficulty();
-    const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+    const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
     let currentDifficulty = genesisBlock.difficulty;
     let totalIndexed = 0;
 
     if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
       await DifficultyAdjustmentsRepository.$saveAdjustments({
-        time: genesisBlock.time,
+        time: genesisBlock.timestamp,
         height: 0,
         difficulty: currentDifficulty,
         adjustment: 0.0,
@@ -458,7 +461,7 @@ class Mining {
   /**
    * Create a link between blocks and the latest price at when they were mined
    */
-  public async $indexBlockPrices() {
+  public async $indexBlockPrices(): Promise<void> {
     if (this.blocksPriceIndexingRunning === true) {
       return;
     }
@@ -519,6 +522,41 @@ class Mining {
     this.blocksPriceIndexingRunning = false;
   }
 
+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/pools-parser.ts

@@ -1,289 +1,161 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
-import BlocksRepository from '../repositories/BlocksRepository';
-
-interface Pool {
-  name: string;
-  link: string;
-  regexes: string[];
-  addresses: string[];
-  slug: string;
-}
+import PoolsRepository from '../repositories/PoolsRepository';
+import { PoolTag } from '../mempool.interfaces';
+import diskCache from './disk-cache';
 
 class PoolsParser {
   miningPools: any[] = [];
   unknownPool: any = {
+    'id': 0,
     'name': 'Unknown',
     'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
     'regexes': '[]',
     'addresses': '[]',
     'slug': 'unknown'
   };
-  slugWarnFlag = false;
+  private uniqueLogs: string[] = [];
+
+  private uniqueLog(loggerFunction: any, msg: string): void {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
+  public setMiningPools(pools): void {
+    for (const pool of pools) {
+      pool.regexes = pool.tags;
+      pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+      delete(pool.tags);
+    }
+    this.miningPools = pools;
+  }
 
   /**
-   * Parse the pools.json file, consolidate the data and dump it into the database
+   * Populate our db with updated mining pool definition
+   * @param pools 
    */
-  public async migratePoolsJson(poolsJson: object): Promise<void> {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return;
-    }
+  public async migratePoolsJson(): Promise<void> {
+    // We also need to wipe the backend cache to make sure we don't serve blocks with
+    // the wrong mining pool (usually happen with unknown blocks)
+    diskCache.wipeCache();
 
-    // First we save every entries without paying attention to pool duplication
-    const poolsDuplicated: Pool[] = [];
+    await this.$insertUnknownPool();
 
-    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
-    for (let i = 0; i < coinbaseTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>coinbaseTags[i][1]).name,
-        'link': (<Pool>coinbaseTags[i][1]).link,
-        'regexes': [coinbaseTags[i][0]],
-        'addresses': [],
-        'slug': ''
-      });
-    }
-    const addressesTags = Object.entries(poolsJson['payout_addresses']);
-    for (let i = 0; i < addressesTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>addressesTags[i][1]).name,
-        'link': (<Pool>addressesTags[i][1]).link,
-        'regexes': [],
-        'addresses': [addressesTags[i][0]],
-        'slug': ''
-      });
-    }
-
-    // Then, we find unique mining pool names
-    const poolNames: string[] = [];
-    for (let i = 0; i < poolsDuplicated.length; ++i) {
-      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
-        poolNames.push(poolsDuplicated[i].name);
+    for (const pool of this.miningPools) {
+      if (!pool.id) {
+        logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
+        continue;
       }
-    }
-    logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);
 
-    // Get existing pools from the db
-    let existingPools;
-    try {
-      if (config.DATABASE.ENABLED === true) {
-        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
+      if (!poolDB) {
+        // New mining pool
+        const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.debug(`Inserting new mining pool ${pool.name}`);
+        await PoolsRepository.$insertNewMiningPool(pool, slug);
+        await this.$deleteUnknownBlocks();
       } else {
-        existingPools = [];
-      }
-    } catch (e) {
-      logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
-      return;
-    }
-
-    this.miningPools = [];
-
-    // Finally, we generate the final consolidated pools data
-    const finalPoolDataAdd: Pool[] = [];
-    const finalPoolDataUpdate: Pool[] = [];
-    const finalPoolDataRename: Pool[] = [];
-    for (let i = 0; i < poolNames.length; ++i) {
-      let allAddresses: string[] = [];
-      let allRegexes: string[] = [];
-      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
-
-      for (let y = 0; y < match.length; ++y) {
-        allAddresses = allAddresses.concat(match[y].addresses);
-        allRegexes = allRegexes.concat(match[y].regexes);
-      }
-
-      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
-
-      let slug: string | undefined;
-      try {
-        slug = poolsJson['slugs'][poolNames[i]];
-      } catch (e) {
-        if (this.slugWarnFlag === false) {
-          logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
-          this.slugWarnFlag = true;
+        if (poolDB.name !== pool.name) {
+          // Pool has been renamed
+          const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+          logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
+          await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
         }
-      }
-
-      if (slug === undefined) {
-        // Only keep alphanumerical
-        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
-      }
-
-      const poolObj = {
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': allRegexes,
-        'addresses': allAddresses,
-        'slug': slug
-      };
-
-      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
-      if (existingPool !== undefined) {
-        // Check if any data was actually updated
-        const equals = (a, b) =>
-          a.length === b.length &&
-          a.every((v, i) => v === b[i]);
-        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
-          finalPoolDataUpdate.push(poolObj);
+        if (poolDB.link !== pool.link) {
+          // Pool link has changed
+          logger.debug(`Updating link for ${pool.name} mining pool`);
+          await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
         }
-      } else {
-        // Double check that if we're not just renaming a pool (same address same regex)
-        const [poolToRename]: any[] = await DB.query(`
-          SELECT * FROM pools
-          WHERE addresses = ? OR regexes = ?`,
-          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
-        );
-        if (poolToRename && poolToRename.length > 0) {
-          // We're actually renaming an existing pool
-          finalPoolDataRename.push({
-            'name': poolObj.name,
-            'link': poolObj.link,
-            'regexes': allRegexes,
-            'addresses': allAddresses,
-            'slug': slug
-          });
-          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
-        } else {
-          logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
-          finalPoolDataAdd.push(poolObj);
+        if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
+          JSON.stringify(pool.regexes) !== poolDB.regexes) {
+          // Pool addresses changed or coinbase tags changed
+          logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
+          await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
+          await this.$deleteBlocksForPool(poolDB);
         }
       }
-
-      this.miningPools.push({
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': JSON.stringify(allRegexes),
-        'addresses': JSON.stringify(allAddresses),
-        'slug': slug
-      });
-    }
-
-    if (config.DATABASE.ENABLED === false) { // Don't run db operations
-      logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
-      return;
-    }
-
-    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
-      finalPoolDataRename.length > 0
-    ) {    
-      logger.debug(`Update pools table now`, logger.tags.mining);
-
-      // Add new mining pools into the database
-      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
-      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
-        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
-      }
-      queryAdd = queryAdd.slice(0, -1) + ';';
-
-      // Updated existing mining pools in the database
-      const updateQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
-        updateQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-          slug='${finalPoolDataUpdate[i].slug}'
-          WHERE name='${finalPoolDataUpdate[i].name}'
-        ;`);
-      }
-
-      // Rename mining pools
-      const renameQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataRename.length; ++i) {
-        renameQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
-            slug='${finalPoolDataRename[i].slug}'
-          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
-            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
-        ;`);
-      }
-
-      try {
-        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
-          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
-        }
-
-        if (finalPoolDataAdd.length > 0) {
-          await DB.query({ sql: queryAdd, timeout: 120000 });
-        }
-        for (const query of updateQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        for (const query of renameQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        await this.insertUnknownPool();
-        logger.info('Mining pools.json import completed', logger.tags.mining);
-      } catch (e) {
-        logger.err(`Cannot import pools in the database`, logger.tags.mining);
-        throw e;
-      }
     }
 
-    try {
-      await this.insertUnknownPool();
-    } catch (e) {
-      logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
-      throw e;
-    }
+    logger.info('Mining pools-v2.json import completed');
   }
 
   /**
    * Manually add the 'unknown pool'
    */
-  private async insertUnknownPool() {
+  public async $insertUnknownPool(): Promise<void> {
+    if (!config.DATABASE.ENABLED) {
+      return;
+    }
+
     try {
       const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
       if (rows.length === 0) {
         await DB.query({
-          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
-          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
+          VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
         `});
       } else {
         await DB.query(`UPDATE pools
-          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
+          SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
           regexes='[]', addresses='[]',
-          slug='unknown'
-          WHERE name='Unknown'
+          slug='${this.unknownPool.slug}',
+          unique_id=0
+          WHERE slug='${this.unknownPool.slug}'
         `);
       }
     } catch (e) {
-      logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
+      logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
     }
   }
 
   /**
-   * Delete blocks which needs to be reindexed
+   * Delete indexed blocks for an updated mining pool
+   * 
+   * @param pool 
    */
-   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
     if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
       return;
     }
 
-    const blockCount = await BlocksRepository.$blockCount(null, null);
-    if (blockCount === 0) {
-      return;
-    }
-
-    for (const updatedPool of finalPoolDataUpdate) {
-      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
-      if (pool.length > 0) {
-        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
-        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
-      }
-    }
-
-    // Ignore early days of Bitcoin as there were not mining pool yet
-    logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
+    // Ignore early days of Bitcoin as there were no mining pool yet
+    const [oldestPoolBlock]: any[] = await DB.query(`
+      SELECT height
+      FROM blocks
+      WHERE pool_id = ?
+      ORDER BY height
+      LIMIT 1`,
+      [pool.id]
+    );
+    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
     const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
+      [unknownPool[0].id]
+    );
+    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ?`,
+      [pool.id]
+    );
+  }
 
-    logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
-    await DB.query(`DELETE FROM hashrates`);
+  private async $deleteUnknownBlocks(): Promise<void> {
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= 130635`,
+      [unknownPool[0].id]
+    );
   }
 }
 

backend/src/api/rbf-cache.ts

@@ -1,46 +1,173 @@
-import { TransactionExtended } from "../mempool.interfaces";
+import { runInNewContext } from "vm";
+import { TransactionExtended, TransactionStripped } from "../mempool.interfaces";
+import { Common } from "./common";
+
+interface RbfTransaction extends TransactionStripped {
+  rbf?: boolean;
+  mined?: boolean;
+}
+
+interface RbfTree {
+  tx: RbfTransaction;
+  time: number;
+  interval?: number;
+  mined?: boolean;
+  fullRbf: boolean;
+  replaces: RbfTree[];
+}
 
 class RbfCache {
-  private replacedBy: { [txid: string]: string; } = {};
-  private replaces: { [txid: string]: string[] } = {};
-  private txs: { [txid: string]: TransactionExtended } = {};
-  private expiring: { [txid: string]: Date } = {};
+  private replacedBy: Map<string, string> = new Map();
+  private replaces: Map<string, string[]> = new Map();
+  private rbfTrees: Map<string, RbfTree> = new Map(); // sequences of consecutive replacements
+  private dirtyTrees: Set<string> = new Set();
+  private treeMap: Map<string, string> = new Map(); // map of txids to sequence ids
+  private txs: Map<string, TransactionExtended> = new Map();
+  private expiring: Map<string, Date> = new Map();
 
   constructor() {
     setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
   }
 
-  public add(replacedTx: TransactionExtended, newTxId: string): void {
-    this.replacedBy[replacedTx.txid] = newTxId;
-    this.txs[replacedTx.txid] = replacedTx;
-    if (!this.replaces[newTxId]) {
-      this.replaces[newTxId] = [];
+  public add(replaced: TransactionExtended[], newTxExtended: TransactionExtended): void {
+    if (!newTxExtended || !replaced?.length) {
+      return;
     }
-    this.replaces[newTxId].push(replacedTx.txid);
+
+    const newTx = Common.stripTransaction(newTxExtended) as RbfTransaction;
+    const newTime = newTxExtended.firstSeen || Date.now();
+    newTx.rbf = newTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
+    this.txs.set(newTx.txid, newTxExtended);
+
+    // maintain rbf trees
+    let fullRbf = false;
+    const replacedTrees: RbfTree[] = [];
+    for (const replacedTxExtended of replaced) {
+      const replacedTx = Common.stripTransaction(replacedTxExtended) as RbfTransaction;
+      replacedTx.rbf = replacedTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
+      this.replacedBy.set(replacedTx.txid, newTx.txid);
+      if (this.treeMap.has(replacedTx.txid)) {
+        const treeId = this.treeMap.get(replacedTx.txid);
+        if (treeId) {
+          const tree = this.rbfTrees.get(treeId);
+          this.rbfTrees.delete(treeId);
+          if (tree) {
+            tree.interval = newTime - tree?.time;
+            replacedTrees.push(tree);
+            fullRbf = fullRbf || tree.fullRbf;
+          }
+        }
+      } else {
+        const replacedTime = replacedTxExtended.firstSeen || Date.now();
+        replacedTrees.push({
+          tx: replacedTx,
+          time: replacedTime,
+          interval: newTime - replacedTime,
+          fullRbf: !replacedTx.rbf,
+          replaces: [],
+        });
+        fullRbf = fullRbf || !replacedTx.rbf;
+        this.txs.set(replacedTx.txid, replacedTxExtended);
+      }
+    }
+    const treeId = replacedTrees[0].tx.txid;
+    const newTree = {
+      tx: newTx,
+      time: newTxExtended.firstSeen || Date.now(),
+      fullRbf,
+      replaces: replacedTrees
+    };
+    this.rbfTrees.set(treeId, newTree);
+    this.updateTreeMap(treeId, newTree);
+    this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid));
+    this.dirtyTrees.add(treeId);
   }
 
   public getReplacedBy(txId: string): string | undefined {
-    return this.replacedBy[txId];
+    return this.replacedBy.get(txId);
   }
 
   public getReplaces(txId: string): string[] | undefined {
-    return this.replaces[txId];
+    return this.replaces.get(txId);
   }
 
   public getTx(txId: string): TransactionExtended | undefined {
-    return this.txs[txId];
+    return this.txs.get(txId);
+  }
+
+  public getRbfTree(txId: string): RbfTree | void {
+    return this.rbfTrees.get(this.treeMap.get(txId) || '');
+  }
+
+  // get a paginated list of RbfTrees
+  // ordered by most recent replacement time
+  public getRbfTrees(onlyFullRbf: boolean, after?: string): RbfTree[] {
+    const limit = 25;
+    const trees: RbfTree[] = [];
+    const used = new Set<string>();
+    const replacements: string[][] = Array.from(this.replacedBy).reverse();
+    const afterTree = after ? this.treeMap.get(after) : null;
+    let ready = !afterTree;
+    for (let i = 0; i < replacements.length && trees.length <= limit - 1; i++) {
+      const txid = replacements[i][1];
+      const treeId = this.treeMap.get(txid) || '';
+      if (treeId === afterTree) {
+        ready = true;
+      } else if (ready) {
+        if (!used.has(treeId)) {
+          const tree = this.rbfTrees.get(treeId);
+          used.add(treeId);
+          if (tree && (!onlyFullRbf || tree.fullRbf)) {
+            trees.push(tree);
+          }
+        }
+      }
+    }
+    return trees;
+  }
+
+  // get map of rbf trees that have been updated since the last call
+  public getRbfChanges(): { trees: {[id: string]: RbfTree }, map: { [txid: string]: string }} {
+    const changes: { trees: {[id: string]: RbfTree }, map: { [txid: string]: string }} = {
+      trees: {},
+      map: {},
+    };
+    this.dirtyTrees.forEach(id => {
+      const tree = this.rbfTrees.get(id);
+      if (tree) {
+        changes.trees[id] = tree;
+        this.getTransactionsInTree(tree).forEach(tx => {
+          changes.map[tx.txid] = id;
+        });
+      }
+    });
+    this.dirtyTrees = new Set();
+    return changes;
+  }
+
+  public mined(txid): void {
+    const treeId = this.treeMap.get(txid);
+    if (treeId && this.rbfTrees.has(treeId)) {
+      const tree = this.rbfTrees.get(treeId);
+      if (tree) {
+        this.setTreeMined(tree, txid);
+        tree.mined = true;
+        this.dirtyTrees.add(treeId);
+      }
+    }
+    this.evict(txid);
   }
 
   // flag a transaction as removed from the mempool
   public evict(txid): void {
-    this.expiring[txid] = new Date(Date.now() + 1000 * 86400); // 24 hours
+    this.expiring.set(txid, new Date(Date.now() + 1000 * 86400)); // 24 hours
   }
 
   private cleanup(): void {
     const currentDate = new Date();
     for (const txid in this.expiring) {
-      if (this.expiring[txid] < currentDate) {
-        delete this.expiring[txid];
+      if ((this.expiring.get(txid) || 0) < currentDate) {
+        this.expiring.delete(txid);
         this.remove(txid);
       }
     }
@@ -48,18 +175,49 @@ class RbfCache {
 
   // remove a transaction & all previous versions from the cache
   private remove(txid): void {
-    // don't remove a transaction while a newer version remains in the mempool
-    if (this.replaces[txid] && !this.replacedBy[txid]) {
-      const replaces = this.replaces[txid];
-      delete this.replaces[txid];
-      for (const tx of replaces) {
+    // don't remove a transaction if a newer version remains in the mempool
+    if (!this.replacedBy.has(txid)) {
+      const replaces = this.replaces.get(txid);
+      this.replaces.delete(txid);
+      this.treeMap.delete(txid);
+      this.txs.delete(txid);
+      this.expiring.delete(txid);
+      for (const tx of (replaces || [])) {
         // recursively remove prior versions from the cache
-        delete this.replacedBy[tx];
-        delete this.txs[tx];
+        this.replacedBy.delete(tx);
+        // if this is the id of a tree, remove that too
+        if (this.treeMap.get(tx) === tx) {
+          this.rbfTrees.delete(tx);
+        }
         this.remove(tx);
       }
     }
   }
+
+  private updateTreeMap(newId: string, tree: RbfTree): void {
+    this.treeMap.set(tree.tx.txid, newId);
+    tree.replaces.forEach(subtree => {
+      this.updateTreeMap(newId, subtree);
+    });
+  }
+
+  private getTransactionsInTree(tree: RbfTree, txs: RbfTransaction[] = []): RbfTransaction[] {
+    txs.push(tree.tx);
+    tree.replaces.forEach(subtree => {
+      this.getTransactionsInTree(subtree, txs);
+    });
+    return txs;
+  }
+
+  private setTreeMined(tree: RbfTree, txid: string): void {
+    if (tree.tx.txid === txid) {
+      tree.tx.mined = true;
+    } else {
+      tree.replaces.forEach(subtree => {
+        this.setTreeMined(subtree, txid);
+      });
+    }
+  }
 }
 
 export default new RbfCache();

backend/src/api/transaction-utils.ts

@@ -14,6 +14,7 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)

backend/src/api/websocket-handler.ts

@@ -8,7 +8,6 @@ import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
 import mempoolBlocks from './mempool-blocks';
-import fiatConversion from './fiat-conversion';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import config from '../config';
@@ -20,6 +19,7 @@ import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import Audit from './audit';
 import { deepClone } from '../utils/clone';
+import priceUpdater from '../tasks/price-updater';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -131,6 +131,14 @@ class WebsocketHandler {
             }
           }
 
+          if (parsedMessage && parsedMessage['track-rbf'] !== undefined) {
+            if (['all', 'fullRbf'].includes(parsedMessage['track-rbf'])) {
+              client['track-rbf'] = parsedMessage['track-rbf'];
+            } else {
+              client['track-rbf'] = false;
+            }
+          }
+
           if (parsedMessage.action === 'init') {
             const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
             if (!_blocks) {
@@ -214,7 +222,7 @@ class WebsocketHandler {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
       'blocks': _blocks,
-      'conversions': fiatConversion.getConversionRates(),
+      'conversions': priceUpdater.latestPrices,
       'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
@@ -264,6 +272,13 @@ class WebsocketHandler {
     const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
     const da = difficultyAdjustment.getDifficultyAdjustment();
     memPool.handleRbfTransactions(rbfTransactions);
+    const rbfChanges = rbfCache.getRbfChanges();
+    let rbfReplacements;
+    let fullRbfReplacements;
+    if (Object.keys(rbfChanges.trees).length) {
+      rbfReplacements = rbfCache.getRbfTrees(false);
+      fullRbfReplacements = rbfCache.getRbfTrees(true);
+    }
     const recommendedFees = feeApi.getRecommendedFee();
 
     this.wss.clients.forEach(async (client) => {
@@ -384,16 +399,17 @@ class WebsocketHandler {
           response['utxoSpent'] = outspends;
         }
 
-        if (rbfTransactions[client['track-tx']]) {
-          for (const rbfTransaction in rbfTransactions) {
-            if (client['track-tx'] === rbfTransaction) {
-              response['rbfTransaction'] = {
-                txid: rbfTransactions[rbfTransaction].txid,
-              };
-              break;
-            }
+        const rbfReplacedBy = rbfCache.getReplacedBy(client['track-tx']);
+        if (rbfReplacedBy) {
+          response['rbfTransaction'] = {
+            txid: rbfReplacedBy,
           }
         }
+
+        const rbfChange = rbfChanges.map[client['track-tx']];
+        if (rbfChange) {
+          response['rbfInfo'] = rbfChanges.trees[rbfChange];
+        }
       }
 
       if (client['track-mempool-block'] >= 0) {
@@ -406,6 +422,12 @@ class WebsocketHandler {
         }
       }
 
+      if (client['track-rbf'] === 'all' && rbfReplacements) {
+        response['rbfLatest'] = rbfReplacements;
+      } else if (client['track-rbf'] === 'fullRbf' && fullRbfReplacements) {
+        response['rbfLatest'] = fullRbfReplacements;
+      }
+
       if (Object.keys(response).length) {
         client.send(JSON.stringify(response));
       }
@@ -419,49 +441,51 @@ class WebsocketHandler {
 
     const _memPool = memPool.getMempool();
 
-    let projectedBlocks;
-    // template calculation functions have mempool side effects, so calculate audits using
-    // a cloned copy of the mempool if we're running a different algorithm for mempool updates
-    const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
-    if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
-      projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
-    } else {
-      projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
-    }
+    if (config.MEMPOOL.AUDIT) {
+      let projectedBlocks;
+      // template calculation functions have mempool side effects, so calculate audits using
+      // a cloned copy of the mempool if we're running a different algorithm for mempool updates
+      const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
+      if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+        projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
+      } else {
+        projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
+      }
 
-    if (Common.indexingEnabled() && memPool.isInSync()) {
-      const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
-      const matchRate = Math.round(score * 100 * 100) / 100;
+      if (Common.indexingEnabled() && memPool.isInSync()) {
+        const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+        const matchRate = Math.round(score * 100 * 100) / 100;
 
-      const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
-        return {
-          txid: tx.txid,
-          vsize: tx.vsize,
-          fee: tx.fee ? Math.round(tx.fee) : 0,
-          value: tx.value,
-        };
-      }) : [];
+        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
+          return {
+            txid: tx.txid,
+            vsize: tx.vsize,
+            fee: tx.fee ? Math.round(tx.fee) : 0,
+            value: tx.value,
+          };
+        }) : [];
 
-      BlocksSummariesRepository.$saveTemplate({
-        height: block.height,
-        template: {
-          id: block.id,
-          transactions: stripped
+        BlocksSummariesRepository.$saveTemplate({
+          height: block.height,
+          template: {
+            id: block.id,
+            transactions: stripped
+          }
+        });
+
+        BlocksAuditsRepository.$saveAudit({
+          time: block.timestamp,
+          height: block.height,
+          hash: block.id,
+          addedTxs: added,
+          missingTxs: censored,
+          freshTxs: fresh,
+          matchRate: matchRate,
+        });
+
+        if (block.extras) {
+          block.extras.matchRate = matchRate;
         }
-      });
-
-      BlocksAuditsRepository.$saveAudit({
-        time: block.timestamp,
-        height: block.height,
-        hash: block.id,
-        addedTxs: added,
-        missingTxs: censored,
-        freshTxs: fresh,
-        matchRate: matchRate,
-      });
-
-      if (block.extras) {
-        block.extras.matchRate = matchRate;
       }
     }
 
@@ -470,7 +494,7 @@ class WebsocketHandler {
     for (const txId of txIds) {
       delete _memPool[txId];
       removed.push(txId);
-      rbfCache.evict(txId);
+      rbfCache.mined(txId);
     }
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {

backend/src/config.ts

@@ -19,7 +19,6 @@ interface IConfig {
     MEMPOOL_BLOCKS_AMOUNT: number;
     INDEXING_BLOCKS_AMOUNT: number;
     BLOCKS_SUMMARIES_INDEXING: boolean;
-    PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
     EXTERNAL_MAX_RETRY: number;
@@ -29,9 +28,11 @@ interface IConfig {
     AUTOMATIC_BLOCK_REINDEXING: boolean;
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
+    AUDIT: boolean;
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
     CPFP_INDEXING: boolean;
+    MAX_BLOCKS_BULK_QUERY: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -140,7 +141,6 @@ const defaults: IConfig = {
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
     'BLOCKS_SUMMARIES_INDEXING': false,
-    'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [],
     'EXTERNAL_MAX_RETRY': 1,
@@ -148,11 +148,13 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
-    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'AUDIT': false,
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
     'CPFP_INDEXING': false,
+    'MAX_BLOCKS_BULK_QUERY': 0,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/database.ts

@@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
 
   private checkDBFlag() {
     if (config.DATABASE.ENABLED === false) {
-      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+      const stack = new Error().stack;
+      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
     }
   }
 

backend/src/index.ts

@@ -10,7 +10,6 @@ import memPool from './api/mempool';
 import diskCache from './api/disk-cache';
 import statistics from './api/statistics/statistics';
 import websocketHandler from './api/websocket-handler';
-import fiatConversion from './api/fiat-conversion';
 import bisq from './api/bisq/bisq';
 import bisqMarkets from './api/bisq/markets';
 import logger from './logger';
@@ -36,6 +35,9 @@ import liquidRoutes from './api/liquid/liquid.routes';
 import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
+import priceUpdater from './tasks/price-updater';
+import chainTips from './api/chain-tips';
+import { AxiosError } from 'axios';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -78,6 +80,21 @@ class Server {
   async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
+    if (config.DATABASE.ENABLED) {
+      await DB.checkDbConnection();
+      try {
+        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
+          await databaseMigration.$blocksReindexingTruncate();
+        }
+        await databaseMigration.$initializeOrMigrateDatabase();
+        if (Common.indexingEnabled()) {
+          await indexer.$resetHashratesIndexingState();
+        }
+      } catch (e) {
+        throw new Error(e instanceof Error ? e.message : 'Error');
+      }
+    }
+
     this.app
       .use((req: Request, res: Response, next: NextFunction) => {
         res.setHeader('Access-Control-Allow-Origin', '*');
@@ -87,34 +104,21 @@ class Server {
       .use(express.text({ type: ['text/plain', 'application/base64'] }))
       ;
 
+    if (config.DATABASE.ENABLED) {
+      await priceUpdater.$initializeLatestPriceWithDb();
+    }
+
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
 
     this.setUpWebsocketHandling();
 
+    await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
     if (config.MEMPOOL.ENABLED) {
       diskCache.loadMempoolCache();
     }
 
-    if (config.DATABASE.ENABLED) {
-      await DB.checkDbConnection();
-      try {
-        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
-          const tables = process.env.npm_config_reindex.split(',');
-          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
-          await Common.sleep$(5000);
-          await databaseMigration.$truncateIndexedData(tables);
-        }
-        await databaseMigration.$initializeOrMigrateDatabase();
-        if (Common.indexingEnabled()) {
-          await indexer.$resetHashratesIndexingState();
-        }
-      } catch (e) {
-        throw new Error(e instanceof Error ? e.message : 'Error');
-      }
-    }
-
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
       statistics.startStatistics();
     }
@@ -127,7 +131,8 @@ class Server {
       }
     }
 
-    fiatConversion.startService();
+    priceUpdater.$run();
+    await chainTips.updateOrphanedBlocks();
 
     this.setUpHttpApiRoutes();
 
@@ -167,22 +172,30 @@ class Server {
           logger.debug(msg);
         }
       }
-      await poolsUpdater.updatePoolsJson();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
-    } catch (e) {
-      const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
+    } catch (e: any) {
+      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
+      loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
+      if (e?.stack) {
+        loggerMsg += ` Stack trace: ${e.stack}`;
+      }
+      // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
+      // From the second Exception, `logger.warn` the Exception and increase the retry delay
+      // Maximum retry delay is 60 seconds
       if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
         logger.debug(loggerMsg);
       }
-      logger.debug(JSON.stringify(e));
+      if (e instanceof AxiosError) {
+        logger.debug(`AxiosError: ${e?.message}`);
+      }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
       this.currentBackendRetryInterval *= 2;
       this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
@@ -193,8 +206,8 @@ class Server {
     try {
       await fundingTxFetcher.$init();
       await networkSyncService.$startService();
-      await forensicsService.$startService();
       await lightningStatsUpdater.$startService();
+      await forensicsService.$startService();
     } catch(e) {
       logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
       await Common.sleep$(1000 * 60);
@@ -221,7 +234,7 @@ class Server {
       memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
       blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
     }
-    fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
+    priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
   

backend/src/indexer.ts

@@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 
+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
+
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
   tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];
 
-  public reindex() {
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
+  }
+
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
     if (Common.indexingEnabled()) {
       this.runIndexer = true;
     }
   }
 
-  public async runSingleTask(task: 'blocksPrices') {
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
     if (!Common.indexingEnabled()) {
       return;
     }
@@ -28,20 +77,27 @@ class Indexer {
       this.tasksRunning.push(task);
       const lastestPriceId = await PricesRepository.$getLatestPriceId();
       if (priceUpdater.historyInserted === false || lastestPriceId === null) {
-        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
         setTimeout(() => {
-          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
           this.runSingleTask('blocksPrices');
         }, 10000);
       } else {
-        logger.debug(`Blocks prices indexer will run now`)
+        logger.debug(`Blocks prices indexer will run now`);
         await mining.$indexBlockPrices();
-        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
       }
     }
+
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
   }
 
-  public async $run() {
+  public async $run(): Promise<void> {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
     ) {
@@ -57,7 +113,9 @@ class Indexer {
     this.runIndexer = false;
     this.indexerRunning = true;
 
-    logger.debug(`Running mining indexer`);
+    logger.info(`Running mining indexer`);
+
+    await this.checkAvailableCoreIndexes();
 
     try {
       await priceUpdater.$run();
@@ -93,7 +151,7 @@ class Indexer {
     setTimeout(() => this.reindex(), runEvery);
   }
 
-  async $resetHashratesIndexingState() {
+  async $resetHashratesIndexingState(): Promise<void> {
     try {
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);

backend/src/mempool.interfaces.ts

@@ -1,8 +1,10 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
-import { HeapNode } from "./utils/pairing-heap";
+import { OrphanedBlock } from './api/chain-tips';
+import { HeapNode } from './utils/pairing-heap';
 
 export interface PoolTag {
-  id: number; // mysql row id
+  id: number;
+  uniqueId: number;
   name: string;
   link: string;
   regexes: string; // JSON array
@@ -16,6 +18,7 @@ export interface PoolInfo {
   link: string;
   blockCount: number;
   slug: string;
+  avgMatchRate: number | null;
 }
 
 export interface PoolStats extends PoolInfo {
@@ -63,6 +66,7 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -144,23 +148,44 @@ export interface TransactionStripped {
 }
 
 export interface BlockExtension {
-  totalFees?: number;
-  medianFee?: number;
-  feeRange?: number[];
-  reward?: number;
-  coinbaseTx?: TransactionMinerInfo;
-  matchRate?: number;
-  pool?: {
-    id: number;
+  totalFees: number;
+  medianFee: number; // median fee rate
+  feeRange: number[]; // fee rate percentiles
+  reward: number;
+  matchRate: number | null;
+  pool: {
+    id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
     name: string;
     slug: string;
   };
-  avgFee?: number;
-  avgFeeRate?: number;
-  coinbaseRaw?: string;
-  usd?: number | null;
+  avgFee: number;
+  avgFeeRate: number;
+  coinbaseRaw: string;
+  orphans: OrphanedBlock[] | null;
+  coinbaseAddress: string | null;
+  coinbaseSignature: string | null;
+  coinbaseSignatureAscii: string | null;
+  virtualSize: number;
+  avgTxSize: number;
+  totalInputs: number;
+  totalOutputs: number;
+  totalOutputAmt: number;
+  medianFeeAmt: number | null; // median fee in sats
+  feePercentiles: number[] | null, // fee percentiles in sats
+  segwitTotalTxs: number;
+  segwitTotalSize: number;
+  segwitTotalWeight: number;
+  header: string;
+  utxoSetChange: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize: number | null;
+  totalInputAmt: number | null;
 }
 
+/**
+ * Note: Everything that is added in here will be automatically returned through
+ * /api/v1/block and /api/v1/blocks APIs
+ */
 export interface BlockExtended extends IEsploraApi.Block {
   extras: BlockExtension;
 }

backend/src/repositories/BlocksRepository.ts

@@ -1,8 +1,7 @@
-import { BlockExtended, BlockPrice } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
-import { prepareBlock } from '../utils/blocks-utils';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
@@ -10,27 +9,90 @@ import BlocksSummariesRepository from './BlocksSummariesRepository';
 import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
 import bitcoinClient from '../api/bitcoin/bitcoin-client';
 import config from '../config';
+import chainTips from '../api/chain-tips';
+import blocks from '../api/blocks';
+import BlocksAuditsRepository from './BlocksAuditsRepository';
+
+const BLOCK_DB_FIELDS = `
+  blocks.hash AS id,
+  blocks.height,
+  blocks.version,
+  UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
+  blocks.bits,
+  blocks.nonce,
+  blocks.difficulty,
+  blocks.merkle_root,
+  blocks.tx_count,
+  blocks.size,
+  blocks.weight,
+  blocks.previous_block_hash AS previousblockhash,
+  UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
+  blocks.fees AS totalFees,
+  blocks.median_fee AS medianFee,
+  blocks.fee_span AS feeRange,
+  blocks.reward,
+  pools.unique_id AS poolId,
+  pools.name AS poolName,
+  pools.slug AS poolSlug,
+  blocks.avg_fee AS avgFee,
+  blocks.avg_fee_rate AS avgFeeRate,
+  blocks.coinbase_raw AS coinbaseRaw,
+  blocks.coinbase_address AS coinbaseAddress,
+  blocks.coinbase_signature AS coinbaseSignature,
+  blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
+  blocks.avg_tx_size AS avgTxSize,
+  blocks.total_inputs AS totalInputs,
+  blocks.total_outputs AS totalOutputs,
+  blocks.total_output_amt AS totalOutputAmt,
+  blocks.median_fee_amt AS medianFeeAmt,
+  blocks.fee_percentiles AS feePercentiles,
+  blocks.segwit_total_txs AS segwitTotalTxs,
+  blocks.segwit_total_size AS segwitTotalSize,
+  blocks.segwit_total_weight AS segwitTotalWeight,
+  blocks.header,
+  blocks.utxoset_change AS utxoSetChange,
+  blocks.utxoset_size AS utxoSetSize,
+  blocks.total_input_amt AS totalInputAmts
+`;
 
 class BlocksRepository {
   /**
    * Save indexed block data in the database
    */
   public async $saveBlockInDatabase(block: BlockExtended) {
+    const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
+    const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
+
     try {
       const query = `INSERT INTO blocks(
-        height,           hash,                blockTimestamp, size,
-        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id,          fees,                fee_span,       median_fee,
-        reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        height,             hash,                blockTimestamp,    size,
+        weight,             tx_count,            coinbase_raw,      difficulty,
+        pool_id,            fees,                fee_span,          median_fee,
+        reward,             version,             bits,              nonce,
+        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        median_timestamp,   header,              coinbase_address,
+        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        median_fee_amt,     coinbase_signature_ascii
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        FROM_UNIXTIME(?), ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?
       )`;
 
+      const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
+      if (!poolDbId) {
+        throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
+      }
+
       const params: any[] = [
         block.height,
         block.id,
@@ -40,7 +102,7 @@ class BlocksRepository {
         block.tx_count,
         block.extras.coinbaseRaw,
         block.difficulty,
-        block.extras.pool?.id, // Should always be set to something
+        poolDbId.id,
         block.extras.totalFees,
         JSON.stringify(block.extras.feeRange),
         block.extras.medianFee,
@@ -52,19 +114,63 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
+        block.mediantime,
+        block.extras.header,
+        block.extras.coinbaseAddress,
+        truncatedCoinbaseSignature,
+        block.extras.utxoSetSize,
+        block.extras.utxoSetChange,
+        block.extras.avgTxSize,
+        block.extras.totalInputs,
+        block.extras.totalOutputs,
+        block.extras.totalInputAmt,
+        block.extras.totalOutputAmt,
+        block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
+        block.extras.segwitTotalTxs,
+        block.extras.segwitTotalSize,
+        block.extras.segwitTotalWeight,
+        block.extras.medianFeeAmt,
+        truncatedCoinbaseSignatureAscii,
       ];
 
       await DB.query(query, params);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
+        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
       } else {
-        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
+        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
         throw e;
       }
     }
   }
 
+  /**
+   * Save newly indexed data from core coinstatsindex
+   * 
+   * @param utxoSetSize 
+   * @param totalInputAmt 
+   */
+  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
+    totalInputAmt: number
+  ) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET utxoset_size = ?, total_input_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        utxoSetSize,
+        totalInputAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -250,34 +356,17 @@ class BlocksRepository {
   /**
    * Get blocks mined by a specific mining pool
    */
-  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
+  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
     const pool = await PoolsRepository.$getPool(slug);
     if (!pool) {
       throw new Error('This mining pool does not exist ' + escape(slug));
     }
 
     const params: any[] = [];
-    let query = ` SELECT
-      blocks.height,
-      hash as id,
-      UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-      size,
-      weight,
-      tx_count,
-      coinbase_raw,
-      difficulty,
-      fees,
-      fee_span,
-      median_fee,
-      reward,
-      version,
-      bits,
-      nonce,
-      merkle_root,
-      previous_block_hash as previousblockhash,
-      avg_fee,
-      avg_fee_rate
+    let query = `
+      SELECT ${BLOCK_DB_FIELDS}
       FROM blocks
+      JOIN pools ON blocks.pool_id = pools.id
       WHERE pool_id = ?`;
     params.push(pool.id);
 
@@ -290,11 +379,11 @@ class BlocksRepository {
       LIMIT 10`;
 
     try {
-      const [rows] = await DB.query(query, params);
+      const [rows]: any[] = await DB.query(query, params);
 
       const blocks: BlockExtended[] = [];
-      for (const block of <object[]>rows) {
-        blocks.push(prepareBlock(block));
+      for (const block of rows) {
+        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
       }
 
       return blocks;
@@ -307,46 +396,21 @@ class BlocksRepository {
   /**
    * Get one block by height
    */
-  public async $getBlockByHeight(height: number): Promise<object | null> {
+  public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
     try {
-      const [rows]: any[] = await DB.query(`SELECT
-        blocks.height,
-        hash,
-        hash as id,
-        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
-        pools.id as pool_id,
-        pools.name as pool_name,
-        pools.link as pool_link,
-        pools.slug as pool_slug,
-        pools.addresses as pool_addresses,
-        pools.regexes as pool_regexes,
-        fees,
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
+      const [rows]: any[] = await DB.query(`
+        SELECT ${BLOCK_DB_FIELDS}
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
-        WHERE blocks.height = ${height}
-      `);
+        WHERE blocks.height = ?`,
+        [height]
+      );
 
       if (rows.length <= 0) {
         return null;
       }
 
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
+      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -359,10 +423,7 @@ class BlocksRepository {
   public async $getBlockByHash(hash: string): Promise<object | null> {
     try {
       const query = `
-        SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
-        pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
-        pools.addresses as pool_addresses, pools.regexes as pool_regexes,
-        previous_block_hash as previousblockhash
+        SELECT ${BLOCK_DB_FIELDS}
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
         WHERE hash = ?;
@@ -372,9 +433,8 @@ class BlocksRepository {
       if (rows.length <= 0) {
         return null;
       }
-
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
+ 
+      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
     } catch (e) {
       logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -465,8 +525,15 @@ class BlocksRepository {
   public async $validateChain(): Promise<boolean> {
     try {
       const start = new Date().getTime();
-      const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
+      const [blocks]: any[] = await DB.query(`
+        SELECT
+          height,
+          hash,
+          previous_block_hash,
+          UNIX_TIMESTAMP(blockTimestamp) AS timestamp
+        FROM blocks
+        ORDER BY height
+      `);
 
       let partialMsg = false;
       let idx = 1;
@@ -694,7 +761,6 @@ class BlocksRepository {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
-    return [];
   }
 
   /**
@@ -741,7 +807,7 @@ class BlocksRepository {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`
+        query += ` (${price.height}, ${price.priceId}),`;
       }
       query = query.slice(0, -1);
       await DB.query(query);
@@ -754,6 +820,132 @@ class BlocksRepository {
       }
     }
   }
+
+  /**
+   * Get all indexed blocsk with missing coinstatsindex data
+   */
+  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash
+        FROM blocks
+        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
+          (utxoset_size IS NULL OR total_input_amt IS NULL)
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save indexed median fee to avoid recomputing it later
+   * 
+   * @param id 
+   * @param feePercentiles 
+   */
+  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?`,
+        [JSON.stringify(feePercentiles), feePercentiles[3], id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Convert a mysql row block into a BlockExtended. Note that you
+   * must provide the correct field into dbBlk object param
+   * 
+   * @param dbBlk 
+   */
+  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
+    const blk: Partial<BlockExtended> = {};
+    const extras: Partial<BlockExtension> = {};
+
+    // IEsploraApi.Block
+    blk.id = dbBlk.id;
+    blk.height = dbBlk.height;
+    blk.version = dbBlk.version;
+    blk.timestamp = dbBlk.timestamp;
+    blk.bits = dbBlk.bits;
+    blk.nonce = dbBlk.nonce;
+    blk.difficulty = dbBlk.difficulty;
+    blk.merkle_root = dbBlk.merkle_root;
+    blk.tx_count = dbBlk.tx_count;
+    blk.size = dbBlk.size;
+    blk.weight = dbBlk.weight;
+    blk.previousblockhash = dbBlk.previousblockhash;
+    blk.mediantime = dbBlk.mediantime;
+    
+    // BlockExtension
+    extras.totalFees = dbBlk.totalFees;
+    extras.medianFee = dbBlk.medianFee;
+    extras.feeRange = JSON.parse(dbBlk.feeRange);
+    extras.reward = dbBlk.reward;
+    extras.pool = {
+      id: dbBlk.poolId,
+      name: dbBlk.poolName,
+      slug: dbBlk.poolSlug,
+    };
+    extras.avgFee = dbBlk.avgFee;
+    extras.avgFeeRate = dbBlk.avgFeeRate;
+    extras.coinbaseRaw = dbBlk.coinbaseRaw;
+    extras.coinbaseAddress = dbBlk.coinbaseAddress;
+    extras.coinbaseSignature = dbBlk.coinbaseSignature;
+    extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
+    extras.avgTxSize = dbBlk.avgTxSize;
+    extras.totalInputs = dbBlk.totalInputs;
+    extras.totalOutputs = dbBlk.totalOutputs;
+    extras.totalOutputAmt = dbBlk.totalOutputAmt;
+    extras.medianFeeAmt = dbBlk.medianFeeAmt;
+    extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
+    extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
+    extras.segwitTotalSize = dbBlk.segwitTotalSize;
+    extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
+    extras.header = dbBlk.header,
+    extras.utxoSetChange = dbBlk.utxoSetChange;
+    extras.utxoSetSize = dbBlk.utxoSetSize;
+    extras.totalInputAmt = dbBlk.totalInputAmt;
+    extras.virtualSize = dbBlk.weight / 4.0;
+
+    // Re-org can happen after indexing so we need to always get the
+    // latest state from core
+    extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
+
+    // Match rate is not part of the blocks table, but it is part of APIs so we must include it
+    extras.matchRate = null;
+    if (config.MEMPOOL.AUDIT) {
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id);
+      if (auditScore != null) {
+        extras.matchRate = auditScore.matchRate;
+      }
+    }
+
+    // If we're missing block summary related field, check if we can populate them on the fly now
+    if (Common.blocksSummariesIndexingEnabled() &&
+      (extras.medianFeeAmt === null || extras.feePercentiles === null))
+    {
+      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
+      if (extras.feePercentiles === null) {
+        const block = await bitcoinClient.getBlock(dbBlk.id, 2);
+        const summary = blocks.summarizeBlock(block);
+        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+        extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
+      }
+      if (extras.feePercentiles !== null) {
+        extras.medianFeeAmt = extras.feePercentiles[3];
+      }
+    }
+
+    blk.extras = <BlockExtension>extras;
+    return <BlockExtended>blk;
+  }
 }
 
 export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -80,6 +80,48 @@ class BlocksSummariesRepository {
       logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0] ?? 0, // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
+        fees[fees.length - 1] ?? 0, // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();

backend/src/repositories/DifficultyAdjustmentsRepository.ts

@@ -1,5 +1,4 @@
 import { Common } from '../api/common';
-import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { IndexedDifficultyAdjustment } from '../mempool.interfaces';

backend/src/repositories/PoolsRepository.ts

@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
@@ -9,7 +10,7 @@ class PoolsRepository {
    * Get all pools tagging info
    */
   public async $getPools(): Promise<PoolTag[]> {
-    const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
+    const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
     return <PoolTag[]>rows;
   }
 
@@ -17,7 +18,11 @@ class PoolsRepository {
    * Get unknown pool tagging info
    */
   public async $getUnknownPool(): Promise<PoolTag> {
-    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
+    if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
+      await poolsParser.$insertUnknownPool();
+      [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
+    }
     return <PoolTag>rows[0];
   }
 
@@ -27,16 +32,25 @@ class PoolsRepository {
   public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
+    let query = `
+      SELECT
+        COUNT(blocks.height) As blockCount,
+          pool_id AS poolId,
+          pools.name AS name,
+          pools.link AS link,
+          slug,
+          AVG(blocks_audits.match_rate) AS avgMatchRate
       FROM blocks
-      JOIN pools on pools.id = pool_id`;
+      JOIN pools on pools.id = pool_id
+      LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
+    `;
 
     if (interval) {
       query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
     }
 
     query += ` GROUP BY pool_id
-      ORDER BY COUNT(height) DESC`;
+      ORDER BY COUNT(blocks.height) DESC`;
 
     try {
       const [rows] = await DB.query(query);
@@ -50,7 +64,7 @@ class PoolsRepository {
   /**
    * Get basic pool info and block count between two timestamp
    */
-   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+  public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
     const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
       FROM pools
       LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
@@ -66,9 +80,9 @@ class PoolsRepository {
   }
 
   /**
-   * Get mining pool statistics for one pool
+   * Get a mining pool info
    */
-   public async $getPool(slug: string): Promise<PoolTag | null> {
+  public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
     const query = `
       SELECT *
       FROM pools
@@ -81,10 +95,12 @@ class PoolsRepository {
         return null;
       }
 
-      rows[0].regexes = JSON.parse(rows[0].regexes);
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
       if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools.json only contains mainnet addresses
-      } else {
+        rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
+      } else if (parse) {
         rows[0].addresses = JSON.parse(rows[0].addresses);
       }
 
@@ -94,6 +110,116 @@ class PoolsRepository {
       throw e;
     }
   }
+
+  /**
+   * Get a mining pool info by its unique id
+   */
+  public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.unique_id = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [id]);
+
+      if (rows.length < 1) {
+        return null;
+      }
+
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+        rows[0].addresses = []; // pools.json only contains mainnet addresses
+      } else if (parse) {
+        rows[0].addresses = JSON.parse(rows[0].addresses);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Insert a new mining pool in the database
+   * 
+   * @param pool 
+   */
+  public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO pools
+        SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`,
+        [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Rename an existing mining pool
+   * 
+   * @param dbId
+   * @param newSlug
+   * @param newName 
+   */
+  public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET slug = ?, name = ?
+        WHERE id = ?`,
+        [newSlug, newName, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update an exisiting mining pool link
+   * 
+   * @param dbId 
+   * @param newLink 
+   */
+  public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET link = ?
+        WHERE id = ?`,
+        [newLink, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+  }
+
+  /**
+   * Update an existing mining pool addresses or coinbase tags
+   * 
+   * @param dbId 
+   * @param addresses 
+   * @param regexes 
+   */
+  public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET addresses = ?, regexes = ?
+        WHERE id = ?`,
+        [JSON.stringify(addresses), JSON.stringify(regexes), dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
 }
 
 export default new PoolsRepository();

backend/src/repositories/PricesRepository.ts

@@ -1,15 +1,59 @@
 import DB from '../database';
 import logger from '../logger';
-import { Prices } from '../tasks/price-updater';
+import { IConversionRates } from '../mempool.interfaces';
+import priceUpdater from '../tasks/price-updater';
+
+export interface ApiPrice {
+  time?: number,
+  USD: number,
+  EUR: number,
+  GBP: number,
+  CAD: number,
+  CHF: number,
+  AUD: number,
+  JPY: number,
+}
+
+export interface ExchangeRates {
+  USDEUR: number,
+  USDGBP: number,
+  USDCAD: number,
+  USDCHF: number,
+  USDAUD: number,
+  USDJPY: number,
+}
+
+export interface Conversion {
+  prices: ApiPrice[],
+  exchangeRates: ExchangeRates;
+}
+
+export const MAX_PRICES = {
+  USD: 100000000,
+  EUR: 100000000,
+  GBP: 100000000,
+  CAD: 100000000,
+  CHF: 100000000,
+  AUD: 100000000,
+  JPY: 10000000000,
+};
 
 class PricesRepository {
-  public async $savePrices(time: number, prices: Prices): Promise<void> {
+  public async $savePrices(time: number, prices: IConversionRates): Promise<void> {
     if (prices.USD === -1) {
-      // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
-      // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
+      // Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues
+      // As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine
       return;
     }
 
+    // Sanity check
+    for (const currency of Object.keys(prices)) {
+      if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry"
+        logger.info(`Ignore BTC${currency} price of ${prices[currency]}`);
+        prices[currency] = 0;
+      }
+    }
+    
     try {
       await DB.query(`
         INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
@@ -23,22 +67,22 @@ class PricesRepository {
   }
 
   public async $getOldestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
   public async $getLatestPriceId(): Promise<number | null> {
-    const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].id : null;
   }
 
   public async $getLatestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
   public async $getPricesTimes(): Promise<number[]> {
-    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time`);
     return times.map(time => time.time);
   }
 
@@ -46,6 +90,86 @@ class PricesRepository {
     const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
     return times;
   }
+
+  public async $getLatestConversionRates(): Promise<any> {
+    const [rates]: any[] = await DB.query(`
+      SELECT USD, EUR, GBP, CAD, CHF, AUD, JPY
+      FROM prices
+      ORDER BY time DESC
+      LIMIT 1`
+    );
+    if (!rates || rates.length === 0) {
+      return priceUpdater.getEmptyPricesObj();
+    }
+    return rates[0];
+  }
+
+  public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
+    try {
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        WHERE UNIX_TIMESTAMP(time) < ?
+        ORDER BY time DESC
+        LIMIT 1`,
+        [timestamp]
+      );
+      if (!rates) {
+        throw Error(`Cannot get single historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      const latestPrice = await this.$getLatestConversionRates();
+      const exchangeRates: ExchangeRates = {
+        USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
+        USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
+        USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
+        USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
+        USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
+        USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
+      };
+
+      return {
+        prices: rates,
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
+
+  public async $getHistoricalPrices(): Promise<Conversion | null> {
+    try {
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        ORDER BY time DESC
+      `);
+      if (!rates) {
+        throw Error(`Cannot get average historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      const latestPrice: ApiPrice = rates[0];
+      const exchangeRates: ExchangeRates = {
+        USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
+        USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
+        USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
+        USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
+        USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
+        USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
+      };
+
+      return {
+        prices: rates,
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
 }
 
 export default new PricesRepository();

backend/src/rpc-api/commands.ts

@@ -88,5 +88,7 @@ module.exports = {
   verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange'
-}
+  walletPassphraseChange: 'walletpassphrasechange',
+  getTxoutSetinfo: 'gettxoutsetinfo',
+  getIndexInfo: 'getindexinfo',
+};

backend/src/tasks/lightning/network-sync.service.ts

@@ -72,7 +72,7 @@ class NetworkSyncService {
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
-      node.last_update = Math.max(node.last_update, latestUpdated);
+      node.last_update = Math.max(node.last_update ?? 0, latestUpdated);
 
       await nodesApi.$saveNode(node);
       graphNodesPubkeys.push(node.pub_key);

backend/src/tasks/pools-updater.ts

@@ -8,7 +8,7 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
 
 /**
- * Maintain the most recent version of pools.json
+ * Maintain the most recent version of pools-v2.json
  */
 class PoolsUpdater {
   lastRun: number = 0;
@@ -31,14 +31,8 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
-    } else {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
-    }
-
     try {
-      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
       if (githubSha === undefined) {
         return;
       }
@@ -47,32 +41,57 @@ class PoolsUpdater {
         this.currentSha = await this.getShaFromDb();
       }
 
-      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
       if (this.currentSha !== undefined && this.currentSha === githubSha) {
         return;
       }
 
+      // See backend README for more details about the mining pools update process
+      if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
+        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
+        !process.env.npm_config_update_pools // We're not manually updating mining pool
+      ) {
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        return;
+      }
+
+      const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
       if (this.currentSha === undefined) {
-        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
         return;
       }
-      await poolsParser.migratePoolsJson(poolsJson);
-      await this.updateDBSha(githubSha);
-      logger.notice(`PoolsUpdater completed`, logger.tags.mining);
+      poolsParser.setMiningPools(poolsJson);
+
+      if (config.DATABASE.ENABLED === false) { // Don't run db operations
+        logger.info('Mining pools-v2.json import completed (no database)');
+        return;
+      }
+
+      try {
+        await DB.query('START TRANSACTION;');
+        await poolsParser.migratePoolsJson();
+        await this.updateDBSha(githubSha);
+        await DB.query('COMMIT;');
+      } catch (e) {
+        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+        await DB.query('ROLLBACK;');
+      }
+      logger.notice('PoolsUpdater completed');
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err(`PoolsUpdater failed. Will try again in 24h. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
   private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
@@ -81,46 +100,46 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       }
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
   private async getShaFromDb(): Promise<string | undefined> {
     try {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : undefined);
     } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       return undefined;
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from github
+   * Fetch our latest pools-v2.json sha from github
    */
   private async fetchPoolsSha(): Promise<string | undefined> {
     const response = await this.query(this.treeUrl);
 
     if (response !== undefined) {
       for (const file of response['tree']) {
-        if (file['path'] === 'pools.json') {
+        if (file['path'] === 'pools-v2.json') {
           return file['sha'];
         }
       }
     }
 
-    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`, logger.tags.mining);
+    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
     return undefined;
   }
 
   /**
    * Http request wrapper
    */
-  private async query(path): Promise<object | undefined> {
+  private async query(path): Promise<any[] | undefined> {
     type axiosOptions = {
       headers: {
         'User-Agent': string

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -13,7 +13,11 @@ class BitfinexApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['last_price'], 10) : -1;
+    if (response && response['last_price']) {
+      return parseInt(response['last_price'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -13,7 +13,11 @@ class BitflyerApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['ltp'], 10) : -1;
+    if (response && response['ltp']) {
+      return parseInt(response['ltp'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -13,7 +13,11 @@ class CoinbaseApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['data']['amount'], 10) : -1;
+    if (response && response['data'] && response['data']['amount']) {
+      return parseInt(response['data']['amount'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/gemini-api.ts

@@ -13,7 +13,11 @@ class GeminiApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['last'], 10) : -1;
+    if (response && response['last']) {
+      return parseInt(response['last'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/kraken-api.ts

@@ -23,7 +23,14 @@ class KrakenApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
+    const ticker = this.getTicker(currency);
+    if (response && response['result'] && response['result'][ticker] &&
+      response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0
+    ) {
+      return parseInt(response['result'][ticker]['c'][0], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-updater.ts

@@ -1,8 +1,9 @@
 import * as fs from 'fs';
-import path from "path";
+import path from 'path';
 import config from '../config';
 import logger from '../logger';
-import PricesRepository from '../repositories/PricesRepository';
+import { IConversionRates } from '../mempool.interfaces';
+import PricesRepository, { MAX_PRICES } from '../repositories/PricesRepository';
 import BitfinexApi from './price-feeds/bitfinex-api';
 import BitflyerApi from './price-feeds/bitflyer-api';
 import CoinbaseApi from './price-feeds/coinbase-api';
@@ -20,17 +21,7 @@ export interface PriceFeed {
 }
 
 export interface PriceHistory {
-  [timestamp: number]: Prices;
-}
-
-export interface Prices {
-  USD: number;
-  EUR: number;
-  GBP: number;
-  CAD: number;
-  CHF: number;
-  AUD: number;
-  JPY: number;
+  [timestamp: number]: IConversionRates;
 }
 
 class PriceUpdater {
@@ -40,7 +31,8 @@ class PriceUpdater {
   running = false;
   feeds: PriceFeed[] = [];
   currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
-  latestPrices: Prices;
+  latestPrices: IConversionRates;
+  private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
 
   constructor() {
     this.latestPrices = this.getEmptyPricesObj();
@@ -52,7 +44,7 @@ class PriceUpdater {
     this.feeds.push(new GeminiApi());
   }
 
-  public getEmptyPricesObj(): Prices {
+  public getEmptyPricesObj(): IConversionRates {
     return {
       USD: -1,
       EUR: -1,
@@ -64,6 +56,18 @@ class PriceUpdater {
     };
   }
 
+  public setRatesChangedCallback(fn: (rates: IConversionRates) => void) {
+    this.ratesChangedCallback = fn;
+  }
+
+  /**
+   * We execute this function before the websocket initialization since
+   * the websocket init is not done asyncronously
+   */
+  public async $initializeLatestPriceWithDb(): Promise<void> {
+    this.latestPrices = await PricesRepository.$getLatestConversionRates();
+  }
+
   public async $run(): Promise<void> {
     if (this.running === true) {
       return;
@@ -76,10 +80,9 @@ class PriceUpdater {
     }
 
     try {
+      await this.$updatePrice();
       if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
         await this.$insertHistoricalPrices();
-      } else {
-        await this.$updatePrice();
       }
     } catch (e) {
       logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
@@ -112,7 +115,7 @@ class PriceUpdater {
         if (feed.currencies.includes(currency)) {
           try {
             const price = await feed.$fetchPrice(currency);
-            if (price > 0) {
+            if (price > -1 && price < MAX_PRICES[currency]) {
               prices.push(price);
             }
             logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
@@ -127,7 +130,11 @@ class PriceUpdater {
 
       // Compute average price, non weighted
       prices = prices.filter(price => price > 0);
-      this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+      if (prices.length === 0) {
+        this.latestPrices[currency] = -1;
+      } else {
+        this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+      }
     }
 
     logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
@@ -144,6 +151,10 @@ class PriceUpdater {
       }
     }
 
+    if (this.ratesChangedCallback) {
+      this.ratesChangedCallback(this.latestPrices);
+    }
+
     this.lastRun = new Date().getTime() / 1000;
   }
 
@@ -213,7 +224,7 @@ class PriceUpdater {
 
     // Group them by timestamp and currency, for example
     // grouped[123456789]['USD'] = [1, 2, 3, 4];
-    const grouped: Object = {};
+    const grouped: any = {};
     for (const historicalEntry of historicalPrices) {
       for (const time in historicalEntry) {
         if (existingPriceTimes.includes(parseInt(time, 10))) {
@@ -228,8 +239,8 @@ class PriceUpdater {
 
         for (const currency of this.currencies) {
           const price = historicalEntry[time][currency];
-          if (price > 0) {
-            grouped[time][currency].push(parseInt(price, 10));
+          if (price > -1 && price < MAX_PRICES[currency]) {
+            grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
           }
         }
       }
@@ -238,7 +249,7 @@ class PriceUpdater {
     // Average prices and insert everything into the db
     let totalInserted = 0;
     for (const time in grouped) {
-      const prices: Prices = this.getEmptyPricesObj();
+      const prices: IConversionRates = this.getEmptyPricesObj();
       for (const currency in grouped[time]) {
         if (grouped[time][currency].length === 0) {
           continue;

backend/src/utils/blocks-utils.ts

@@ -1,33 +0,0 @@
-import { BlockExtended } from '../mempool.interfaces';
-
-export function prepareBlock(block: any): BlockExtended {
-  return <BlockExtended>{
-    id: block.id ?? block.hash, // hash for indexed block
-    timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
-    height: block.height,
-    version: block.version,
-    bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
-    nonce: block.nonce,
-    difficulty: block.difficulty,
-    merkle_root: block.merkle_root ?? block.merkleroot,
-    tx_count: block.tx_count ?? block.nTx,
-    size: block.size,
-    weight: block.weight,
-    previousblockhash: block.previousblockhash,
-    extras: {
-      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
-      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
-      feeRange: block.feeRange ?? block.fee_span,
-      reward: block.reward ?? block?.extras?.reward,
-      totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
-      avgFee: block?.extras?.avgFee ?? block.avg_fee,
-      avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
-      pool: block?.extras?.pool ?? (block?.pool_id ? {
-        id: block.pool_id,
-        name: block.pool_name,
-        slug: block.pool_slug,
-      } : undefined),
-      usd: block?.extras?.usd ?? block.usd ?? null,
-    }
-  };
-}

contributors/Arooba-git.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: Arooba-git

docker/README.md

@@ -101,17 +101,17 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "BLOCKS_SUMMARIES_INDEXING": false,
-    "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
+    "EXTERNAL_ASSETS": [],
     "STDOUT_LOG_MIN_PRIORITY": "info",
     "INDEXING_BLOCKS_AMOUNT": false,
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
+    "MAX_BLOCKS_BULK_QUERY": 0,
   },
 ```
 
@@ -132,7 +132,6 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_INITIAL_BLOCKS_AMOUNT: ""
       MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: ""
       MEMPOOL_BLOCKS_SUMMARIES_INDEXING: ""
-      MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: ""
       MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
@@ -143,6 +142,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_ADVANCED_GBT_AUDIT: ""
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
+      MAX_BLOCKS_BULK_QUERY: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -13,7 +13,6 @@
     "BLOCK_WEIGHT_UNITS": __MEMPOOL_BLOCK_WEIGHT_UNITS__,
     "INITIAL_BLOCKS_AMOUNT": __MEMPOOL_INITIAL_BLOCKS_AMOUNT__,
     "MEMPOOL_BLOCKS_AMOUNT": __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__,
-    "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
     "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
     "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
     "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__,
@@ -23,9 +22,11 @@
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
     "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
     "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
+    "AUDIT": __MEMPOOL_AUDIT__,
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
-    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__
+    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL__MAX_BLOCKS_BULK_QUERY__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -16,7 +16,6 @@ __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
 __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__=${MEMPOOL_BLOCKS_SUMMARIES_INDEXING:=false}
-__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
 __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
 __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
@@ -25,11 +24,13 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
-__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
+__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
+__MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
+__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -128,7 +129,6 @@ sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}
 sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json
-sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json
@@ -139,9 +139,11 @@ sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT_
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

docker/frontend/entrypoint.sh

@@ -31,9 +31,11 @@ __LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
 __BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets}
 __MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
 __LIGHTNING__=${LIGHTNING:=false}
+__AUDIT__=${AUDIT:=false}
 __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
+__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
 
 # Export as environment variables to be used by envsubst
 export __TESTNET_ENABLED__
@@ -55,9 +57,11 @@ export __LIQUID_WEBSITE_URL__
 export __BISQ_WEBSITE_URL__
 export __MINING_DASHBOARD__
 export __LIGHTNING__
+export __AUDIT__
 export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
 export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
 export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
+export __HISTORICAL_PRICE__
 
 folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
 echo ${folder}

frontend/.gitignore

@@ -54,6 +54,7 @@ src/resources/assets-testnet.json
 src/resources/assets-testnet.minimal.json
 src/resources/pools.json
 src/resources/mining-pools/*
+src/resources/*.mp4
 
 # environment config
 mempool-frontend-config.json

frontend/.tx/config

@@ -1,7 +1,9 @@
 [main]
 host = https://www.transifex.com
 
-[mempool.frontend-src-locale-messages-xlf--master]
+[o:mempool:p:mempool:r:frontend-src-locale-messages-xlf--master]
 file_filter = frontend/src/locale/messages.<lang>.xlf
+source_file = frontend/src/locale/messages.en-US.xlf
 source_lang = en-US
-type = XLIFF
+type        = XLIFF
+

frontend/cypress.config.ts

@@ -1,4 +1,4 @@
-import { defineConfig } from 'cypress'
+import { defineConfig } from 'cypress';
 
 export default defineConfig({
   projectId: 'ry4br7',
@@ -12,12 +12,18 @@ export default defineConfig({
   },
   chromeWebSecurity: false,
   e2e: {
-    // We've imported your old cypress plugins here.
-    // You may want to clean this up later by importing these.
-    setupNodeEvents(on, config) {
-      return require('./cypress/plugins/index.js')(on, config)
+    setupNodeEvents(on: any, config: any) {
+      const fs = require('fs');
+      const CONFIG_FILE = 'mempool-frontend-config.json';
+      if (fs.existsSync(CONFIG_FILE)) {
+        let contents = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
+        config.env.BASE_MODULE = contents.BASE_MODULE ? contents.BASE_MODULE : 'mempool';
+      } else {
+        config.env.BASE_MODULE = 'mempool';
+      }
+      return config;
     },
     baseUrl: 'http://localhost:4200',
     specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
   },
-})
+});

frontend/cypress/e2e/bisq/bisq.spec.ts

@@ -1,5 +1,5 @@
 describe('Bisq', () => {
-  const baseModule = Cypress.env("BASE_MODULE");
+  const baseModule = Cypress.env('BASE_MODULE');
   const basePath = '';
 
   beforeEach(() => {
@@ -20,7 +20,7 @@ describe('Bisq', () => {
       cy.waitForSkeletonGone();
     });
 
-    describe("transactions", () => {
+    describe('transactions', () => {
       it('loads the transactions screen', () => {
         cy.visit(`${basePath}`);
         cy.waitForSkeletonGone();
@@ -30,9 +30,9 @@ describe('Bisq', () => {
       });
 
       const filters = [
-        "Asset listing fee", "Blind vote", "Compensation request",
-        "Genesis", "Irregular", "Lockup", "Pay trade fee", "Proof of burn",
-        "Proposal", "Reimbursement request", "Transfer BSQ", "Unlock", "Vote reveal"
+        'Asset listing fee', 'Blind vote', 'Compensation request',
+        'Genesis', 'Irregular', 'Lockup', 'Pay trade fee', 'Proof of burn',
+        'Proposal', 'Reimbursement request', 'Transfer BSQ', 'Unlock', 'Vote reveal'
       ];
       filters.forEach((filter) => {
         it.only(`filters the transaction screen by ${filter}`, () => {
@@ -49,7 +49,7 @@ describe('Bisq', () => {
         });
       });
 
-      it("filters using multiple criteria", () => {
+      it('filters using multiple criteria', () => {
         const filters = ['Proposal', 'Lockup', 'Unlock'];
         cy.visit(`${basePath}/transactions`);
         cy.waitForSkeletonGone();

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -1,5 +1,5 @@
 describe('Liquid', () => {
-  const baseModule = Cypress.env("BASE_MODULE");
+  const baseModule = Cypress.env('BASE_MODULE');
   const basePath = '';
 
   beforeEach(() => {

frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts

@@ -1,5 +1,5 @@
 describe('Liquid Testnet', () => {
-  const baseModule = Cypress.env("BASE_MODULE");
+  const baseModule = Cypress.env('BASE_MODULE');
   const basePath = '/testnet';
 
   beforeEach(() => {

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -1,6 +1,6 @@
-import { emitMempoolInfo, dropWebSocket } from "../../support/websocket";
+import { emitMempoolInfo, dropWebSocket } from '../../support/websocket';
 
-const baseModule = Cypress.env("BASE_MODULE");
+const baseModule = Cypress.env('BASE_MODULE');
 
 
 //Credit: https://github.com/bahmutov/cypress-examples/blob/6cedb17f83a3bb03ded13cf1d6a3f0656ca2cdf5/docs/recipes/overlapping-elements.md
@@ -64,7 +64,7 @@ describe('Mainnet', () => {
     it('loads the status screen', () => {
       cy.visit('/status');
       cy.get('#mempool-block-0').should('be.visible');
-      cy.get('[id^="bitcoin-block-"]').should('have.length', 8);
+      cy.get('[id^="bitcoin-block-"]').should('have.length', 22);
       cy.get('.footer').should('be.visible');
       cy.get('.row > :nth-child(1)').invoke('text').then((text) => {
         expect(text).to.match(/Incoming transactions.* vB\/s/);
@@ -219,11 +219,11 @@ describe('Mainnet', () => {
     describe('blocks navigation', () => {
 
       describe('keyboard events', () => {
-        it('loads first blockchain blocks visible and keypress arrow right', () => {
+        it('loads first blockchain block visible and keypress arrow right', () => {
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('.blockchain-blocks-0 > a').click().then(() => {
+          cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
             cy.waitForPageIdle();
@@ -233,11 +233,11 @@ describe('Mainnet', () => {
           });
         });
 
-        it('loads first blockchain blocks visible and keypress arrow left', () => {
+        it('loads first blockchain block visible and keypress arrow left', () => {
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('.blockchain-blocks-0 > a').click().then(() => {
+          cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
             cy.waitForPageIdle();
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
@@ -246,11 +246,11 @@ describe('Mainnet', () => {
           });
         });
 
-        it('loads last blockchain blocks and keypress arrow right', () => {
+        it.skip('loads last blockchain block and keypress arrow right', () => { //Skip for now as "last" doesn't really work with infinite scrolling
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('.blockchain-blocks-4 > a').click().then(() => {
+          cy.get('bitcoin-block-offset-0-index-7').click().then(() => {
             cy.waitForPageIdle();
 
             // block 6
@@ -309,7 +309,7 @@ describe('Mainnet', () => {
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('.blockchain-blocks-0 > a').click().then(() => {
+          cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
             cy.waitForPageIdle();
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
@@ -339,14 +339,14 @@ describe('Mainnet', () => {
       cy.visit('/');
       cy.waitForSkeletonGone();
 
-      cy.changeNetwork("testnet");
-      cy.changeNetwork("signet");
-      cy.changeNetwork("mainnet");
+      cy.changeNetwork('testnet');
+      cy.changeNetwork('signet');
+      cy.changeNetwork('mainnet');
     });
 
     it.skip('loads the dashboard with the skeleton blocks', () => {
       cy.mockMempoolSocket();
-      cy.visit("/");
+      cy.visit('/');
       cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');

frontend/cypress/e2e/mainnet/mining.spec.ts

@@ -1,4 +1,4 @@
-const baseModule = Cypress.env("BASE_MODULE");
+const baseModule = Cypress.env('BASE_MODULE');
 
 describe('Mainnet - Mining Features', () => {
   beforeEach(() => {

frontend/cypress/e2e/signet/signet.spec.ts

@@ -1,6 +1,6 @@
-import { emitMempoolInfo } from "../../support/websocket";
+import { emitMempoolInfo } from '../../support/websocket';
 
-const baseModule = Cypress.env("BASE_MODULE");
+const baseModule = Cypress.env('BASE_MODULE');
 
 describe('Signet', () => {
   beforeEach(() => {
@@ -25,7 +25,7 @@ describe('Signet', () => {
 
     it.skip('loads the dashboard with the skeleton blocks', () => {
       cy.mockMempoolSocket();
-      cy.visit("/signet");
+      cy.visit('/signet');
       cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');
@@ -35,7 +35,7 @@ describe('Signet', () => {
 
       emitMempoolInfo({
         'params': {
-          "network": "signet"
+          'network': 'signet'
         }
       });
 

frontend/cypress/e2e/testnet/testnet.spec.ts

@@ -1,6 +1,6 @@
-import { confirmAddress, emitMempoolInfo, sendWsMock, showNewTx, startTrackingAddress } from "../../support/websocket";
+import { emitMempoolInfo } from '../../support/websocket';
 
-const baseModule = Cypress.env("BASE_MODULE");
+const baseModule = Cypress.env('BASE_MODULE');
 
 describe('Testnet', () => {
   beforeEach(() => {
@@ -25,7 +25,7 @@ describe('Testnet', () => {
 
     it.skip('loads the dashboard with the skeleton blocks', () => {
       cy.mockMempoolSocket();
-      cy.visit("/testnet");
+      cy.visit('/testnet');
       cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');

frontend/cypress/plugins/index.js

@@ -1,13 +0,0 @@
-const fs = require('fs');
-
-const CONFIG_FILE = 'mempool-frontend-config.json';
-
-module.exports = (on, config) => {
-  if (fs.existsSync(CONFIG_FILE)) {
-    let contents = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
-    config.env.BASE_MODULE = contents.BASE_MODULE ? contents.BASE_MODULE : 'mempool';
-  } else {
-    config.env.BASE_MODULE = 'mempool';
-  }
-  return config;
-}

frontend/mempool-frontend-config.sample.json

@@ -17,8 +17,10 @@
   "LIQUID_WEBSITE_URL": "https://liquid.network",
   "BISQ_WEBSITE_URL": "https://bisq.markets",
   "MINING_DASHBOARD": true,
+  "AUDIT": false,
   "MAINNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
-  "LIGHTNING": false
+  "LIGHTNING": false,
+  "HISTORICAL_PRICE": true
 }

frontend/package-lock.json

@@ -58,7 +58,7 @@
       },
       "optionalDependencies": {
         "@cypress/schematic": "^2.4.0",
-        "cypress": "^12.3.0",
+        "cypress": "^12.7.0",
         "cypress-fail-on-console-error": "~4.0.2",
         "cypress-wait-until": "^1.7.2",
         "mock-socket": "~9.1.5",
@@ -7010,9 +7010,9 @@
       "peer": true
     },
     "node_modules/cypress": {
-      "version": "12.3.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.3.0.tgz",
-      "integrity": "sha512-ZQNebibi6NBt51TRxRMYKeFvIiQZ01t50HSy7z/JMgRVqBUey3cdjog5MYEbzG6Ktti5ckDt1tfcC47lmFwXkw==",
+      "version": "12.7.0",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.7.0.tgz",
+      "integrity": "sha512-7rq+nmhzz0u6yabCFyPtADU2OOrYt6pvUau9qV7xyifJ/hnsaw/vkr0tnLlcuuQKUAOC1v1M1e4Z0zG7S0IAvA==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
@@ -7033,7 +7033,7 @@
         "commander": "^5.1.0",
         "common-tags": "^1.8.0",
         "dayjs": "^1.10.4",
-        "debug": "^4.3.2",
+        "debug": "^4.3.4",
         "enquirer": "^2.3.6",
         "eventemitter2": "6.4.7",
         "execa": "4.1.0",
@@ -7159,6 +7159,23 @@
         "node": ">= 6"
       }
     },
+    "node_modules/cypress/node_modules/debug": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "optional": true,
+      "dependencies": {
+        "ms": "2.1.2"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/cypress/node_modules/execa": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",
@@ -22276,9 +22293,9 @@
       "peer": true
     },
     "cypress": {
-      "version": "12.3.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.3.0.tgz",
-      "integrity": "sha512-ZQNebibi6NBt51TRxRMYKeFvIiQZ01t50HSy7z/JMgRVqBUey3cdjog5MYEbzG6Ktti5ckDt1tfcC47lmFwXkw==",
+      "version": "12.7.0",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.7.0.tgz",
+      "integrity": "sha512-7rq+nmhzz0u6yabCFyPtADU2OOrYt6pvUau9qV7xyifJ/hnsaw/vkr0tnLlcuuQKUAOC1v1M1e4Z0zG7S0IAvA==",
       "optional": true,
       "requires": {
         "@cypress/request": "^2.88.10",
@@ -22298,7 +22315,7 @@
         "commander": "^5.1.0",
         "common-tags": "^1.8.0",
         "dayjs": "^1.10.4",
-        "debug": "^4.3.2",
+        "debug": "^4.3.4",
         "enquirer": "^2.3.6",
         "eventemitter2": "6.4.7",
         "execa": "4.1.0",
@@ -22382,6 +22399,15 @@
           "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==",
           "optional": true
         },
+        "debug": {
+          "version": "4.3.4",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+          "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+          "optional": true,
+          "requires": {
+            "ms": "2.1.2"
+          }
+        },
         "execa": {
           "version": "4.1.0",
           "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",

frontend/package.json

@@ -110,7 +110,7 @@
   },
   "optionalDependencies": {
     "@cypress/schematic": "^2.4.0",
-    "cypress": "^12.3.0",
+    "cypress": "^12.7.0",
     "cypress-fail-on-console-error": "~4.0.2",
     "cypress-wait-until": "^1.7.2",
     "mock-socket": "~9.1.5",
@@ -119,4 +119,4 @@
   "scarfSettings": {
     "enabled": false
   }
-}
+}

frontend/proxy.conf.staging.js

@@ -3,8 +3,8 @@ const fs = require('fs');
 let PROXY_CONFIG = require('./proxy.conf');
 
 PROXY_CONFIG.forEach(entry => {
-  entry.target = entry.target.replace("mempool.space", "mempool-staging.tk7.mempool.space");
-  entry.target = entry.target.replace("liquid.network", "liquid-staging.tk7.mempool.space");
+  entry.target = entry.target.replace("mempool.space", "mempool-staging.fra.mempool.space");
+  entry.target = entry.target.replace("liquid.network", "liquid-staging.fra.mempool.space");
   entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space");
 });
 

frontend/src/app/app-routing.module.ts

@@ -14,6 +14,7 @@ import { TrademarkPolicyComponent } from './components/trademark-policy/trademar
 import { BisqMasterPageComponent } from './components/bisq-master-page/bisq-master-page.component';
 import { PushTransactionComponent } from './components/push-transaction/push-transaction.component';
 import { BlocksList } from './components/blocks-list/blocks-list.component';
+import { RbfList } from './components/rbf-list/rbf-list.component';
 import { LiquidMasterPageComponent } from './components/liquid-master-page/liquid-master-page.component';
 import { AssetGroupComponent } from './components/assets/asset-group/asset-group.component';
 import { AssetsFeaturedComponent } from './components/assets/assets-featured/assets-featured.component';
@@ -56,6 +57,10 @@ let routes: Routes = [
             path: 'blocks',
             component: BlocksList,
           },
+          {
+            path: 'rbf',
+            component: RbfList,
+          },
           {
             path: 'terms-of-service',
             component: TermsOfServiceComponent
@@ -162,6 +167,10 @@ let routes: Routes = [
             path: 'blocks',
             component: BlocksList,
           },
+          {
+            path: 'rbf',
+            component: RbfList,
+          },
           {
             path: 'terms-of-service',
             component: TermsOfServiceComponent
@@ -264,6 +273,10 @@ let routes: Routes = [
         path: 'blocks',
         component: BlocksList,
       },
+      {
+        path: 'rbf',
+        component: RbfList,
+      },
       {
         path: 'terms-of-service',
         component: TermsOfServiceComponent

frontend/src/app/app.constants.ts

@@ -72,22 +72,10 @@ export const chartColors = [
 ];
 
 export const poolsColor = {
-   'foundryusa': '#D81B60',
-   'antpool': '#8E24AA',
-   'f2pool': '#5E35B1',
-   'poolin': '#3949AB',
-   'binancepool': '#1E88E5',
-   'viabtc': '#039BE5',
-   'btccom': '#00897B',
-   'braiinspool': '#00ACC1',
-   'sbicrypto': '#43A047',
-   'marapool': '#7CB342',
-   'luxor': '#C0CA33',
-   'unknown': '#FDD835',
-   'okkong': '#FFB300',
-}
+  'unknown': '#FDD835',
+};
 
- export const feeLevels = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
+export const feeLevels = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
   250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
 
 export interface Language {
@@ -157,3 +145,41 @@ export const specialBlocks = {
     labelEventCompleted: 'Block Subsidy has halved to 3.125 BTC per block',
   }
 };
+
+export const fiatCurrencies = {
+  AUD: {
+    name: 'Australian Dollar',
+    code: 'AUD',
+    indexed: true,
+  },
+  CAD: {
+    name: 'Canadian Dollar',
+    code: 'CAD',
+    indexed: true,
+  },
+  CHF: {
+    name: 'Swiss Franc',
+    code: 'CHF',
+    indexed: true,
+  },
+  EUR: {
+    name: 'Euro',
+    code: 'EUR',
+    indexed: true,
+  },
+  GBP: {
+    name: 'Pound Sterling',
+    code: 'GBP',
+    indexed: true,
+  },
+  JPY: {
+    name: 'Japanese Yen',
+    code: 'JPY',
+    indexed: true,
+  },
+  USD: {
+    name: 'US Dollar',
+    code: 'USD',
+    indexed: true,
+  },
+};

frontend/src/app/app.module.ts

@@ -7,6 +7,7 @@ import { AppComponent } from './components/app/app.component';
 import { ElectrsApiService } from './services/electrs-api.service';
 import { StateService } from './services/state.service';
 import { CacheService } from './services/cache.service';
+import { PriceService } from './services/price.service';
 import { EnterpriseService } from './services/enterprise.service';
 import { WebsocketService } from './services/websocket.service';
 import { AudioService } from './services/audio.service';
@@ -17,6 +18,7 @@ import { StorageService } from './services/storage.service';
 import { HttpCacheInterceptor } from './services/http-cache.interceptor';
 import { LanguageService } from './services/language.service';
 import { FiatShortenerPipe } from './shared/pipes/fiat-shortener.pipe';
+import { FiatCurrencyPipe } from './shared/pipes/fiat-currency.pipe';
 import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe';
 import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe';
 import { AppPreloadingStrategy } from './app.preloading-strategy';
@@ -25,6 +27,7 @@ const providers = [
   ElectrsApiService,
   StateService,
   CacheService,
+  PriceService,
   WebsocketService,
   AudioService,
   SeoService,
@@ -34,6 +37,7 @@ const providers = [
   LanguageService,
   ShortenStringPipe,
   FiatShortenerPipe,
+  FiatCurrencyPipe,
   CapAddressPipe,
   AppPreloadingStrategy,
   { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true }

frontend/src/app/components/about/about.component.html

@@ -13,19 +13,9 @@
     <p i18n>Our mempool and blockchain explorer for the Bitcoin community, focusing on the transaction fee market and multi-layer ecosystem, completely self-hosted without any trusted third-parties.</p>
   </div>
 
-  <div class="social-icons">
-    <a target="_blank" href="https://github.com/mempool/mempool">
-      <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="github" class="svg-inline--fa fa-github fa-w-16 fa-4x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path fill="currentColor" d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path></svg>
-    </a>
-    <a target="_blank" href="https://twitter.com/mempool">
-      <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="twitter" class="svg-inline--fa fa-twitter fa-w-16 fa-4x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"></path></svg>
-    </a>
-    <a target="_blank" href="https://matrix.to/#/#mempool:bitcoin.kyoto">
-      <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="matrix" class="svg-inline--fa fa-matrix fa-w-16 fa-4x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1536 1792"><path fill="currentColor" d="M40.467 163.152v1465.696H145.92V1664H0V128h145.92v35.152zm450.757 464.64v74.14h2.069c19.79-28.356 43.717-50.215 71.483-65.575 27.765-15.656 59.963-23.336 96-23.336 34.56 0 66.165 6.795 94.818 20.086 28.652 13.293 50.216 37.22 65.28 70.893 16.246-23.926 38.4-45.194 66.166-63.507 27.766-18.314 60.848-27.472 98.954-27.472 28.948 0 55.828 3.545 80.64 10.635 24.812 7.088 45.785 18.314 63.508 33.968 17.722 15.656 31.31 35.742 41.354 60.85 9.747 25.107 14.768 55.236 14.768 90.683v366.573h-150.35V865.28c0-18.314-.59-35.741-2.068-51.987-1.476-16.247-5.316-30.426-11.52-42.24-6.499-12.112-15.656-21.563-28.062-28.653-12.405-7.088-29.242-10.634-50.214-10.634-21.268 0-38.4 4.135-51.397 12.112-12.997 8.27-23.336 18.608-30.72 31.901-7.386 12.997-12.407 27.765-14.77 44.602-2.363 16.542-3.84 33.379-3.84 50.216v305.133H692.971v-307.2c0-16.247-.294-32.197-1.18-48.149-.591-15.95-3.84-30.424-9.157-44.011-5.317-13.293-14.178-24.223-26.585-32.197-12.406-7.976-30.425-12.112-54.646-12.112-7.088 0-16.542 1.478-28.062 4.726-11.52 3.25-23.04 9.157-33.968 18.02-10.93 8.86-20.383 21.563-28.063 38.103-7.68 16.543-11.52 38.4-11.52 65.28v317.834H349.44V627.792zm1004.309 1001.056V163.152H1390.08V128H1536v1536h-145.92v-35.152z"/></svg>
-    </a>
-  </div>
+  <video src="/resources/mempool-promo.mp4" poster="/resources/mempool-promo.jpg" controls loop playsinline [autoplay]="true" [muted]="true"></video>
 
-  <div class="enterprise-sponsor">
+  <div class="enterprise-sponsor" id="enterprise-sponsors">
     <h3 i18n="about.sponsors.enterprise.withRocket">Enterprise Sponsors 🚀</h3>
     <div class="wrapper">
       <a href="https://spiral.xyz/" target="_blank" title="Spiral">
@@ -173,7 +163,7 @@
     </div>
   </div>
 
-  <div class="community-sponsor">
+  <div class="community-sponsor" id="community-sponsors">
     <h3 i18n="about.sponsors.withHeart">Community Sponsors ❤️</h3>
 
     <div class="wrapper">
@@ -187,7 +177,7 @@
     </div>
   </div>
 
-  <div class="community-integrations-sponsor">
+  <div class="community-integrations-sponsor" id="community-integrations">
     <h3 i18n="about.community-integrations">Community Integrations</h3>
     <div class="wrapper">
       <a href="https://github.com/getumbrel/umbrel" target="_blank" title="Umbrel">
@@ -281,7 +271,7 @@
     </div>
   </div>
 
-  <div class="alliances">
+  <div class="alliances" id="community-alliances">
     <h3 i18n="about.alliances">Community Alliances</h3>
     <div class="wrapper">
       <a href="https://liquid.net/" title="Liquid Network">
@@ -297,7 +287,7 @@
   </div>
 
   <ng-container *ngIf="translators$ | async | keyvalue as translators else loadingSponsors">
-    <div class="project-translators">
+    <div class="project-translators" id="project-translators">
       <h3 i18n="about.translators">Project Translators</h3>
       <div class="wrapper">
         <ng-template ngFor let-translator [ngForOf]="translators">
@@ -311,7 +301,7 @@
   </ng-container>
 
   <ng-container *ngIf="allContributors$ | async as contributors else loadingSponsors">
-    <div class="contributors">
+    <div class="contributors" id="project-contributors">
       <h3 i18n="about.contributors">Project Contributors</h3>
       <div class="wrapper">
         <ng-template ngFor let-contributor [ngForOf]="contributors.regular">
@@ -323,7 +313,7 @@
       </div>
     </div>
 
-    <div class="maintainers" *ngIf="contributors.core.length">
+    <div class="maintainers" *ngIf="contributors.core.length" id="project-members">
       <h3 i18n="about.project_members">Project Members</h3>
       <div class="wrapper">
         <ng-template ngFor let-contributor [ngForOf]="contributors.core">
@@ -336,7 +326,7 @@
     </div>
   </ng-container>
 
-  <div class="maintainers">
+  <div class="maintainers" id="project-maintainers">
     <h3 i18n="about.maintainers">Project Maintainers</h3>
     <div class="wrapper">
         <a href="https://twitter.com/softsimon_" target="_blank" title="softsimon">
@@ -352,7 +342,7 @@
 
   <div class="copyright">
     <div class="title">
-      Copyright &copy; 2019-2022<br>
+      Copyright &copy; 2019-2023<br>
       The Mempool Open Source Project
     </div>
     <p>
@@ -383,6 +373,17 @@
   <div class="footer-links">
     <a href="/3rdpartylicenses.txt">Third-party Licenses</a>
     <a [routerLink]="['/terms-of-service']" i18n="shared.terms-of-service|Terms of Service">Terms of Service</a>
+    <div class="social-icons">
+      <a target="_blank" href="https://github.com/mempool/mempool">
+        <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="github" class="svg-inline--fa fa-github fa-w-16 fa-2x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path fill="currentColor" d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path></svg>
+      </a>
+      <a target="_blank" href="https://twitter.com/mempool">
+        <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="twitter" class="svg-inline--fa fa-twitter fa-w-16 fa-2x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"></path></svg>
+      </a>
+      <a target="_blank" href="https://matrix.to/#/#mempool:bitcoin.kyoto">
+        <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="matrix" class="svg-inline--fa fa-matrix fa-w-16 fa-2x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1536 1792"><path fill="currentColor" d="M40.467 163.152v1465.696H145.92V1664H0V128h145.92v35.152zm450.757 464.64v74.14h2.069c19.79-28.356 43.717-50.215 71.483-65.575 27.765-15.656 59.963-23.336 96-23.336 34.56 0 66.165 6.795 94.818 20.086 28.652 13.293 50.216 37.22 65.28 70.893 16.246-23.926 38.4-45.194 66.166-63.507 27.766-18.314 60.848-27.472 98.954-27.472 28.948 0 55.828 3.545 80.64 10.635 24.812 7.088 45.785 18.314 63.508 33.968 17.722 15.656 31.31 35.742 41.354 60.85 9.747 25.107 14.768 55.236 14.768 90.683v366.573h-150.35V865.28c0-18.314-.59-35.741-2.068-51.987-1.476-16.247-5.316-30.426-11.52-42.24-6.499-12.112-15.656-21.563-28.062-28.653-12.405-7.088-29.242-10.634-50.214-10.634-21.268 0-38.4 4.135-51.397 12.112-12.997 8.27-23.336 18.608-30.72 31.901-7.386 12.997-12.407 27.765-14.77 44.602-2.363 16.542-3.84 33.379-3.84 50.216v305.133H692.971v-307.2c0-16.247-.294-32.197-1.18-48.149-.591-15.95-3.84-30.424-9.157-44.011-5.317-13.293-14.178-24.223-26.585-32.197-12.406-7.976-30.425-12.112-54.646-12.112-7.088 0-16.542 1.478-28.062 4.726-11.52 3.25-23.04 9.157-33.968 18.02-10.93 8.86-20.383 21.563-28.063 38.103-7.68 16.543-11.52 38.4-11.52 65.28v317.834H349.44V627.792zm1004.309 1001.056V163.152H1390.08V128H1536v1536h-145.92v-35.152z"/></svg>
+      </a>
+    </div>
   </div>
 
   <div class="footer-version" *ngIf="officialMempoolSpace">

frontend/src/app/components/about/about.component.scss

@@ -34,6 +34,13 @@
     padding: 10px 15px 15px;
   }
 
+  video {
+    width: 640px;
+    height: 360px;
+    max-width: 90%;
+    margin-top: 0;
+  }
+
   .social-icons {
     a {
       margin: auto 10px;
@@ -46,6 +53,7 @@
   .maintainers {
     margin-top: 68px;
     margin-bottom: 68px;
+    scroll-margin: 30px;
   }
 
   .maintainers {
@@ -68,8 +76,8 @@
 
   .community-sponsor {
     img {
-      width: 67px;
-      height: 67px;
+      width: 57px;
+      height: 57px;
     }
   }
 
@@ -117,6 +125,7 @@
   .project-translators,
   .community-integrations-sponsor,
   .maintainers {
+    scroll-margin: 30px;
     .wrapper {
       display: inline-block;
       a {
@@ -145,6 +154,13 @@
     }
   }
 
+  .project-translators .wrapper {
+    a img {
+      width: 72px;
+      height: 72px;
+    }
+  }
+
   .copyright {
     text-align: left;
     max-width: 620px;
@@ -179,6 +195,11 @@
         margin: 20px auto 30px;
       }
     }
+    .social-icons {
+      a {
+        margin: 45px 10px;
+      }
+    }
   }
 
   .footer-version {

frontend/src/app/components/about/about.component.ts

@@ -5,9 +5,10 @@ import { StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
 import { ApiService } from '../../services/api.service';
 import { IBackendInfo } from '../../interfaces/websocket.interface';
-import { Router } from '@angular/router';
-import { map } from 'rxjs/operators';
+import { Router, ActivatedRoute } from '@angular/router';
+import { map, tap } from 'rxjs/operators';
 import { ITranslators } from '../../interfaces/node-api.interface';
+import { DOCUMENT } from '@angular/common';
 
 @Component({
   selector: 'app-about',
@@ -31,7 +32,9 @@ export class AboutComponent implements OnInit {
     public stateService: StateService,
     private apiService: ApiService,
     private router: Router,
+    private route: ActivatedRoute,
     @Inject(LOCALE_ID) public locale: string,
+    @Inject(DOCUMENT) private document: Document,
   ) { }
 
   ngOnInit() {
@@ -39,17 +42,21 @@ export class AboutComponent implements OnInit {
     this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`);
     this.websocketService.want(['blocks']);
 
-    this.sponsors$ = this.apiService.getDonation$();
+    this.sponsors$ = this.apiService.getDonation$()
+      .pipe(
+        tap(() => this.goToAnchor())
+      );
     this.translators$ = this.apiService.getTranslators$()
       .pipe(
         map((translators) => {
           for (const t in translators) {
             if (translators[t] === '') {
-              delete translators[t]
+              delete translators[t];
             }
           }
           return translators;
-        })
+        }),
+        tap(() => this.goToAnchor())
       );
     this.allContributors$ = this.apiService.getContributor$().pipe(
       map((contributors) => {
@@ -57,9 +64,24 @@ export class AboutComponent implements OnInit {
           regular: contributors.filter((user) => !user.core_constributor),
           core: contributors.filter((user) => user.core_constributor),
         };
-      })
+      }),
+      tap(() => this.goToAnchor())
     );
   }
+  
+  ngAfterViewInit() {
+    this.goToAnchor();
+  }
+
+  goToAnchor() {
+    setTimeout(() => {
+      if (this.route.snapshot.fragment) {
+        if (this.document.getElementById(this.route.snapshot.fragment)) {
+          this.document.getElementById(this.route.snapshot.fragment).scrollIntoView({behavior: 'smooth'});
+        }
+      }
+    }, 1);
+  }
 
   sponsor(): void {
     if (this.officialMempoolSpace && this.stateService.env.BASE_MODULE === 'mempool') {

frontend/src/app/components/address/address.component.html

@@ -1,4 +1,4 @@
-<div class="container-xl">
+<div class="container-xl" [class.liquid-address]="network === 'liquid' || network === 'liquidtestnet'">
   <div class="title-address">
     <h1 i18n="shared.address">Address</h1>
     <div class="tx-link">
@@ -15,7 +15,7 @@
 
       <div class="row">
         <div class="col-md">
-          <table class="table table-borderless table-striped">
+          <table class="table table-borderless table-striped address-table">
             <tbody>
               <tr *ngIf="addressInfo && addressInfo.unconfidential">
                 <td i18n="address.unconfidential">Unconfidential</td>

frontend/src/app/components/address/address.component.scss

@@ -91,3 +91,20 @@ h1 {
     margin-bottom: 10px;
   }
 }
+
+.liquid-address {
+  .address-table {
+    table-layout: fixed;
+
+    tr td:first-child {
+      width: 170px;
+    }
+    tr td:last-child {
+      width: 80%;
+    }
+  }
+
+  .qrcode-col {
+    flex-grow: 0.5;
+  }
+}

frontend/src/app/components/amount/amount.component.html

@@ -1,7 +1,19 @@
 <ng-container *ngIf="!noFiat && (viewFiat$ | async) && (conversions$ | async) as conversions; else viewFiatVin">
-  <span class="fiat">{{ addPlus && satoshis >= 0 ? '+' : '' }}{{ conversions.USD * (satoshis / 100000000) | currency:'USD':'symbol':'1.2-2' }}</span>
+  <span class="fiat" *ngIf="blockConversion; else noblockconversion">
+    {{ addPlus && satoshis >= 0 ? '+' : '' }}
+    {{
+      (
+        (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ??
+        (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0
+      ) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency
+    }}
+  </span>
+  <ng-template #noblockconversion>
+    <span class="fiat">{{ addPlus && satoshis >= 0 ? '+' : '' }}{{ (conversions ? conversions[currency] : 0) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }}</span>
+  </ng-template>
 </ng-container>
-<ng-template #viewFiatVin>
+
+  <ng-template #viewFiatVin>
   <ng-template [ngIf]="(network === 'liquid' || network === 'liquidtestnet') && (satoshis === undefined || satoshis === null)" [ngIfElse]="default">
     <span i18n="shared.confidential">Confidential</span>
   </ng-template>

frontend/src/app/components/amount/amount.component.ts

@@ -1,6 +1,7 @@
-import { Component, OnInit, OnDestroy, Input, ChangeDetectionStrategy } from '@angular/core';
+import { Component, OnInit, OnDestroy, Input, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core';
 import { StateService } from '../../services/state.service';
 import { Observable, Subscription } from 'rxjs';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-amount',
@@ -10,19 +11,28 @@ import { Observable, Subscription } from 'rxjs';
 })
 export class AmountComponent implements OnInit, OnDestroy {
   conversions$: Observable<any>;
+  currency: string;
   viewFiat$: Observable<boolean>;
   network = '';
 
   stateSubscription: Subscription;
+  currencySubscription: Subscription;
 
   @Input() satoshis: number;
   @Input() digitsInfo = '1.8-8';
   @Input() noFiat = false;
   @Input() addPlus = false;
+  @Input() blockConversion: Price;
 
   constructor(
     private stateService: StateService,
-  ) { }
+    private cd: ChangeDetectorRef,
+  ) {
+    this.currencySubscription = this.stateService.fiatCurrency$.subscribe((fiat) => {
+      this.currency = fiat;
+      this.cd.markForCheck();
+    });
+  }
 
   ngOnInit() {
     this.viewFiat$ = this.stateService.viewFiat$.asObservable();
@@ -34,6 +44,7 @@ export class AmountComponent implements OnInit, OnDestroy {
     if (this.stateSubscription) {
       this.stateSubscription.unsubscribe();
     }
+    this.currencySubscription.unsubscribe();
   }
 
 }

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html

@@ -10,7 +10,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 144" [class.active]="radioGroupForm.get('dateSpan').value === '24h'">
           <input type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]" formControlName="dateSpan"> 24h
         </label>

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-fees-graph/block-fees-graph.component.html

@@ -10,7 +10,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 4320" [class.active]="radioGroupForm.get('dateSpan').value === '1m'">
           <input type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]" formControlName="dateSpan"> 1M
         </label>

frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts

@@ -4,13 +4,14 @@ import { Observable } from 'rxjs';
 import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
 import { ApiService } from '../../services/api.service';
 import { SeoService } from '../../services/seo.service';
-import { formatCurrency, formatNumber, getCurrencySymbol } from '@angular/common';
+import { formatNumber } from '@angular/common';
 import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms';
-import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from '../../shared/graphs.utils';
+import { download, formatterXAxis } from '../../shared/graphs.utils';
 import { StorageService } from '../../services/storage.service';
 import { MiningService } from '../../services/mining.service';
 import { ActivatedRoute } from '@angular/router';
 import { FiatShortenerPipe } from '../../shared/pipes/fiat-shortener.pipe';
+import { FiatCurrencyPipe } from '../../shared/pipes/fiat-currency.pipe';
 
 @Component({
   selector: 'app-block-fees-graph',
@@ -44,6 +45,8 @@ export class BlockFeesGraphComponent implements OnInit {
   timespan = '';
   chartInstance: any = undefined;
 
+  currency: string;
+
   constructor(
     @Inject(LOCALE_ID) public locale: string,
     private seoService: SeoService,
@@ -53,9 +56,11 @@ export class BlockFeesGraphComponent implements OnInit {
     private miningService: MiningService,
     private route: ActivatedRoute,
     private fiatShortenerPipe: FiatShortenerPipe,
+    private fiatCurrencyPipe: FiatCurrencyPipe,
   ) {
     this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' });
     this.radioGroupForm.controls.dateSpan.setValue('1y');
+    this.currency = 'USD';
   }
 
   ngOnInit(): void {
@@ -76,6 +81,7 @@ export class BlockFeesGraphComponent implements OnInit {
       .pipe(
         startWith(this.radioGroupForm.controls.dateSpan.value),
         switchMap((timespan) => {
+          this.isLoading = true;
           this.storageService.setValue('miningWindowPreference', timespan);
           this.timespan = timespan;
           this.isLoading = true;
@@ -84,7 +90,7 @@ export class BlockFeesGraphComponent implements OnInit {
               tap((response) => {
                 this.prepareChartOptions({
                   blockFees: response.body.map(val => [val.timestamp * 1000, val.avgFees / 100000000, val.avgHeight]),
-                  blockFeesUSD: response.body.filter(val => val.USD > 0).map(val => [val.timestamp * 1000, val.avgFees / 100000000 * val.USD, val.avgHeight]),
+                  blockFeesFiat: response.body.filter(val => val[this.currency] > 0).map(val => [val.timestamp * 1000, val.avgFees / 100000000 * val[this.currency], val.avgHeight]),
                 });
                 this.isLoading = false;
               }),
@@ -157,7 +163,7 @@ export class BlockFeesGraphComponent implements OnInit {
             if (tick.seriesIndex === 0) {
               tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.3-3')} BTC<br>`;
             } else if (tick.seriesIndex === 1) {
-              tooltip += `${tick.marker} ${tick.seriesName}: ${formatCurrency(tick.data[1], this.locale, getCurrencySymbol('USD', 'narrow'), 'USD', '1.0-0')}<br>`;
+              tooltip += `${tick.marker} ${tick.seriesName}: ${this.fiatCurrencyPipe.transform(tick.data[1], null, this.currency) }<br>`;
             }
           }
 
@@ -184,7 +190,7 @@ export class BlockFeesGraphComponent implements OnInit {
             icon: 'roundRect',
           },
           {
-            name: 'Fees USD',
+            name: 'Fees ' + this.currency,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {  
               color: 'white',
@@ -216,7 +222,7 @@ export class BlockFeesGraphComponent implements OnInit {
           axisLabel: {
             color: 'rgb(110, 112, 121)',
             formatter: function(val) {
-              return this.fiatShortenerPipe.transform(val);
+              return this.fiatShortenerPipe.transform(val, null, this.currency);
             }.bind(this)
           },
           splitLine: {
@@ -243,8 +249,8 @@ export class BlockFeesGraphComponent implements OnInit {
           legendHoverLink: false,
           zlevel: 1,
           yAxisIndex: 1,
-          name: 'Fees USD',
-          data: data.blockFeesUSD,
+          name: 'Fees ' + this.currency,
+          data: data.blockFeesFiat,
           type: 'line',
           smooth: 0.25,
           symbol: 'none',

frontend/src/app/components/block-overview-graph/block-overview-graph.component.html

@@ -10,5 +10,6 @@
     [cursorPosition]="tooltipPosition"
     [clickable]="!!selectedTx"
     [auditEnabled]="auditHighlighting"
+    [blockConversion]="blockConversion"
   ></app-block-overview-tooltip>
 </div>

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -5,6 +5,7 @@ import BlockScene from './block-scene';
 import TxSprite from './tx-sprite';
 import TxView from './tx-view';
 import { Position } from './sprite-types';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block-overview-graph',
@@ -21,6 +22,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   @Input() mirrorTxid: string | void;
   @Input() unavailable: boolean = false;
   @Input() auditHighlighting: boolean = false;
+  @Input() blockConversion: Price;
   @Output() txClickEvent = new EventEmitter<TransactionStripped>();
   @Output() txHoverEvent = new EventEmitter<string>();
   @Output() readyEvent = new EventEmitter();

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -16,11 +16,11 @@
       </tr>
       <tr>
         <td class="td-width" i18n="dashboard.latest-transactions.amount">Amount</td>
-        <td><app-amount [satoshis]="value"></app-amount></td>
+        <td><app-amount [blockConversion]="blockConversion" [satoshis]="value" [noFiat]="true"></app-amount></td>
       </tr>
       <tr>
         <td class="td-width" i18n="transaction.fee|Transaction fee">Fee</td>
-        <td>{{ fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> &nbsp; <span class="fiat"><app-fiat [value]="fee"></app-fiat></span></td>
+        <td>{{ fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> &nbsp; <span class="fiat"><app-fiat [blockConversion]="blockConversion" [value]="fee"></app-fiat></span></td>
       </tr>
       <tr>
         <td class="td-width" i18n="transaction.fee-rate|Transaction fee rate">Fee rate</td>

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts

@@ -1,6 +1,7 @@
 import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core';
 import { TransactionStripped } from '../../interfaces/websocket.interface';
 import { Position } from '../../components/block-overview-graph/sprite-types.js';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block-overview-tooltip',
@@ -12,6 +13,7 @@ export class BlockOverviewTooltipComponent implements OnChanges {
   @Input() cursorPosition: Position;
   @Input() clickable: boolean;
   @Input() auditEnabled: boolean = false;
+  @Input() blockConversion: Price;
 
   txid = '';
   fee = 0;

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.html

@@ -10,7 +10,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 144" [class.active]="radioGroupForm.get('dateSpan').value === '24h'">
           <input type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]" formControlName="dateSpan"> 24h
         </label>

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html

@@ -11,7 +11,7 @@
     </div>  
   
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 4320" [class.active]="radioGroupForm.get('dateSpan').value === '1m'">
           <input type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]" formControlName="dateSpan"> 1M
         </label>
@@ -31,7 +31,7 @@
           <input type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]" formControlName="dateSpan"> 3Y
         </label>
         <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('dateSpan').value === 'all'">
-          <input type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> ALL
+          <input type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]" formControlName="dateSpan"> ALL
         </label>
       </div>
     </form>

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts

@@ -4,13 +4,14 @@ import { Observable } from 'rxjs';
 import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
 import { ApiService } from '../../services/api.service';
 import { SeoService } from '../../services/seo.service';
-import { formatCurrency, formatNumber, getCurrencySymbol } from '@angular/common';
+import { formatNumber } from '@angular/common';
 import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms';
-import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from '../../shared/graphs.utils';
+import { download, formatterXAxis } from '../../shared/graphs.utils';
 import { MiningService } from '../../services/mining.service';
 import { StorageService } from '../../services/storage.service';
 import { ActivatedRoute } from '@angular/router';
 import { FiatShortenerPipe } from '../../shared/pipes/fiat-shortener.pipe';
+import { FiatCurrencyPipe } from '../../shared/pipes/fiat-currency.pipe';
 
 @Component({
   selector: 'app-block-rewards-graph',
@@ -44,6 +45,8 @@ export class BlockRewardsGraphComponent implements OnInit {
   timespan = '';
   chartInstance: any = undefined;
 
+  currency: string;
+
   constructor(
     @Inject(LOCALE_ID) public locale: string,
     private seoService: SeoService,
@@ -53,7 +56,9 @@ export class BlockRewardsGraphComponent implements OnInit {
     private storageService: StorageService,
     private route: ActivatedRoute,
     private fiatShortenerPipe: FiatShortenerPipe,
+    private fiatCurrencyPipe: FiatCurrencyPipe,
   ) {
+    this.currency = 'USD';
   }
 
   ngOnInit(): void {
@@ -65,7 +70,7 @@ export class BlockRewardsGraphComponent implements OnInit {
     this.route
       .fragment
       .subscribe((fragment) => {
-        if (['3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
+        if (['1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
           this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
         }
       });
@@ -82,7 +87,7 @@ export class BlockRewardsGraphComponent implements OnInit {
               tap((response) => {
                 this.prepareChartOptions({
                   blockRewards: response.body.map(val => [val.timestamp * 1000, val.avgRewards / 100000000, val.avgHeight]),
-                  blockRewardsUSD: response.body.filter(val => val.USD > 0).map(val => [val.timestamp * 1000, val.avgRewards / 100000000 * val.USD, val.avgHeight]),
+                  blockRewardsFiat: response.body.filter(val => val[this.currency] > 0).map(val => [val.timestamp * 1000, val.avgRewards / 100000000 * val[this.currency], val.avgHeight]),
                 });
                 this.isLoading = false;
               }),
@@ -157,7 +162,7 @@ export class BlockRewardsGraphComponent implements OnInit {
             if (tick.seriesIndex === 0) {
               tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.3-3')} BTC<br>`;
             } else if (tick.seriesIndex === 1) {
-              tooltip += `${tick.marker} ${tick.seriesName}: ${formatCurrency(tick.data[1], this.locale, getCurrencySymbol('USD', 'narrow'), 'USD', '1.0-0')}<br>`;
+              tooltip += `${tick.marker} ${tick.seriesName}: ${this.fiatCurrencyPipe.transform(tick.data[1], null, this.currency)}<br>`;
             }
           }
 
@@ -184,7 +189,7 @@ export class BlockRewardsGraphComponent implements OnInit {
             icon: 'roundRect',
           },
           {
-            name: 'Rewards USD',
+            name: 'Rewards ' + this.currency,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {  
               color: 'white',
@@ -228,7 +233,7 @@ export class BlockRewardsGraphComponent implements OnInit {
           axisLabel: {
             color: 'rgb(110, 112, 121)',
             formatter: function(val) {
-              return this.fiatShortenerPipe.transform(val);
+              return this.fiatShortenerPipe.transform(val, null, this.currency);
             }.bind(this)
           },
           splitLine: {
@@ -251,8 +256,8 @@ export class BlockRewardsGraphComponent implements OnInit {
           legendHoverLink: false,
           zlevel: 1,
           yAxisIndex: 1,
-          name: 'Rewards USD',
-          data: data.blockRewardsUSD,
+          name: 'Rewards ' + this.currency,
+          data: data.blockRewardsFiat,
           type: 'line',
           smooth: 0.25,
           symbol: 'none',

frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html

@@ -9,7 +9,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(blockSizesWeightsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 144" [class.active]="radioGroupForm.get('dateSpan').value === '24h'">
           <input type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-sizes-weights' | relativeUrl]" formControlName="dateSpan"> 24h
         </label>

frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}