Merge branch 'master' into nymkappa/feature/historical-rates

2022-05-31 17:35:54 +09:00
parent 4591489e8c f2153edbb4
commit ae567e0cd5
76 changed files with 4169 additions and 4218 deletions
--- a/backend/README.md
+++ b/backend/README.md
@@ -1,22 +1,161 @@
-# Setup backend watchers
+# Mempool Backend

-The backend is static. Typescript scripts are compiled into the `dist` folder and served through a node web server.
+These instructions are mostly intended for developers, but can be used as a basis for personal or small-scale production setups. 

-You can avoid the manual shutdown/recompile/restart command line cycle by using a watcher.
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.

-Make sure you are in the `backend` directory `cd backend`.
+See other ways to set up Mempool on [the main README](/../../#installation-methods).

-1. Install nodemon and ts-node
+Jump to a section in this doc:
+- [Set Up the Backend](#setup)
+- [Development Tips](#development-tips)
+
+## Setup
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:

 ```
-sudo npm install -g ts-node nodemon
+git clone https://github.com/mempool/mempool
+cd mempool
 ```

-2. Run the watcher
+Check out the latest release:

-> Note: You can find your npm global binary folder using `npm -g bin`, where nodemon will be installed.
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Configure Bitcoin Core
+
+Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
+
+```
+txindex=1
+server=1
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+### 3. Configure Electrum Server
+
+[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
+
+**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
+
+### 4. Configure MariaDB
+
+_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
+
+Get MariaDB from your operating system's package manager:
+
+```
+# Debian, Ubuntu, etc.
+apt-get install mariadb-server mariadb-client
+
+# macOS
+brew install mariadb
+mysql.server start
+```
+
+Create a database and grant privileges:
+
+```
+MariaDB [(none)]> drop database mempool;
+Query OK, 0 rows affected (0.00 sec)
+
+MariaDB [(none)]> create database mempool;
+Query OK, 1 row affected (0.00 sec)
+
+MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+Query OK, 0 rows affected (0.00 sec)
+```
+
+### 5. Prepare Mempool Backend
+
+#### Build
+
+_Node.js 16 and npm 7 are recommended._
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd backend
+npm install # add --prod for production
+npm run build
+```
+
+#### Configure
+
+In the backend folder, make a copy of the sample config file:
+
+```
+cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` as needed. 
+
+In particular, make sure:
+- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
+- the correct `BACKEND` is specified in `MEMPOOL`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
+  - "none" if you're not using any Electrum Server
+
+### 6. Run Mempool Backend
+
+Run the Mempool backend:
+
+```
+npm run start
+```
+
+When it's running, you should see output like this:
+
+```
+Mempool updated in 0.189 seconds
+Updating mempool
+Mempool updated in 0.096 seconds
+Updating mempool
+Mempool updated in 0.099 seconds
+Updating mempool
+Calculated fee for transaction 1 / 10
+Calculated fee for transaction 2 / 10
+Calculated fee for transaction 3 / 10
+Calculated fee for transaction 4 / 10
+Calculated fee for transaction 5 / 10
+Calculated fee for transaction 6 / 10
+Calculated fee for transaction 7 / 10
+Calculated fee for transaction 8 / 10
+Calculated fee for transaction 9 / 10
+Calculated fee for transaction 10 / 10
+Mempool updated in 0.243 seconds
+Updating mempool
+```
+
+### 7. Set Up Mempool Frontend
+With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
+
+## Development Tips
+
+### Set Up Backend Watchers
+
+The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
+
+As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
+
+First, install `nodemon` and `ts-node`:
+
+```
+npm install -g ts-node nodemon
+```
+
+Then, run the watcher:

 ```
 nodemon src/index.ts --ignore cache/ --ignore pools.json
 ```

+`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -424,10 +424,7 @@ class Blocks {
    return blockExtended;
  }

-  public async $getBlocksExtras(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-    // Note - This API is breaking if indexing is not available. For now it is okay because we only
-    // use it for the mining pages, and mining pages should not be available if indexing is turned off.
-    // I'll need to fix it before we refactor the block(s) related pages
+  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
    try {
      let currentHeight = fromHeight !== undefined ? fromHeight : this.getCurrentBlockHeight();
      const returnBlocks: BlockExtended[] = [];
@@ -436,25 +433,32 @@ class Blocks {
        return returnBlocks;
      }

+      if (currentHeight === 0 && Common.indexingEnabled()) {
+        currentHeight = await blocksRepository.$mostRecentBlockHeight();
+      }
+
      // Check if block height exist in local cache to skip the hash lookup
      const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
      let startFromHash: string | null = null;
      if (blockByHeight) {
        startFromHash = blockByHeight.id;
-      } else {
+      } else if (!Common.indexingEnabled()) {
        startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
      }

      let nextHash = startFromHash;
      for (let i = 0; i < limit && currentHeight >= 0; i++) {
        let block = this.getBlocks().find((b) => b.height === currentHeight);
-        if (!block && Common.indexingEnabled()) {
+        if (block) {
+          returnBlocks.push(block);
+        } else if (Common.indexingEnabled()) {
          block = await this.$indexBlock(currentHeight);
-        } else if (!block) {
+          returnBlocks.push(block);
+        } else if (nextHash != null) {
          block = prepareBlock(await bitcoinApi.$getBlock(nextHash));
+          nextHash = block.previousblockhash;
+          returnBlocks.push(block);
        }
-        returnBlocks.push(block);
-        nextHash = block.previousblockhash;
        currentHeight--;
      }

--- a/backend/src/api/mining.ts
+++ b/backend/src/api/mining.ts
@@ -89,14 +89,18 @@ class Mining {
    });

    poolsStatistics['pools'] = poolsStats;
-    poolsStatistics['oldestIndexedBlockTimestamp'] = await BlocksRepository.$oldestBlockTimestamp();

    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
    poolsStatistics['blockCount'] = blockCount;

    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
-    const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
-    poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
+
+    try {
+      poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      poolsStatistics['lastEstimatedHashrate'] = 0;
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }

    return poolsStatistics;
  }
@@ -119,7 +123,12 @@ class Mining {
    const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
    const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');

-    const currentEstimatedkHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    let currentEstimatedHashrate = 0;
+    try {
+      currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }

    return {
      pool: pool,
@@ -133,7 +142,7 @@ class Mining {
        '24h': blockCount24h / totalBlock24h,
        '1w': blockCount1w / totalBlock1w,
      },
-      estimatedHashrate: currentEstimatedkHashrate * (blockCount24h / totalBlock24h),
+      estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
      reportedHashrate: null,
    };
  }
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -25,9 +25,6 @@ import databaseMigration from './api/database-migration';
 import syncAssets from './sync-assets';
 import icons from './api/liquid/icons';
 import { Common } from './api/common';
-import mining from './api/mining';
-import HashratesRepository from './repositories/HashratesRepository';
-import BlocksRepository from './repositories/BlocksRepository';
 import poolsUpdater from './tasks/pools-updater';
 import indexer from './indexer';

@@ -315,8 +312,8 @@ class Server {
    }

    this.app
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras', routes.getBlocksExtras)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes))
      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock);

    if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -330,8 +327,6 @@ class Server {
        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus)
        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks)
        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -81,7 +81,7 @@ export interface TransactionStripped {

 export interface BlockExtension {
  totalFees?: number;
-  medianFee?: number; // Actually the median fee rate that we compute ourself
+  medianFee?: number;
  feeRange?: number[];
  reward?: number;
  coinbaseTx?: TransactionMinerInfo;
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -121,6 +121,19 @@ class BlocksRepository {
    }
  }

+  /**
+   * Return most recent block height
+   */
+  public async $mostRecentBlockHeight(): Promise<number> {
+    try {
+      const [row] = await DB.query('SELECT MAX(height) as maxHeight from blocks');
+      return row[0]['maxHeight'];
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
  /**
   * Get blocks count for a period
   */
@@ -475,9 +488,9 @@ class BlocksRepository {
  public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
    try {
      let query = `SELECT
-        CAST(AVG(height) as INT) as avg_height,
+        CAST(AVG(height) as INT) as avgHeight,
        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(fees) as INT) as avg_fees
+        CAST(AVG(fees) as INT) as avgFees
        FROM blocks`;

      if (interval !== null) {
@@ -500,9 +513,9 @@ class BlocksRepository {
  public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
    try {
      let query = `SELECT
-        CAST(AVG(height) as INT) as avg_height,
+        CAST(AVG(height) as INT) as avgHeight,
        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(reward) as INT) as avg_rewards
+        CAST(AVG(reward) as INT) as avgRewards
        FROM blocks`;

      if (interval !== null) {
@@ -525,15 +538,15 @@ class BlocksRepository {
   public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
    try {
      let query = `SELECT
-        CAST(AVG(height) as INT) as avg_height,
+        CAST(AVG(height) as INT) as avgHeight,
        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avg_fee_0,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avg_fee_10,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avg_fee_25,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avg_fee_50,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avg_fee_75,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avg_fee_90,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avg_fee_100
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avgFee_0,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avgFee_10,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avgFee_25,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avgFee_50,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avgFee_75,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avgFee_90,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avgFee_100
      FROM blocks`;

      if (interval !== null) {
@@ -556,9 +569,9 @@ class BlocksRepository {
   public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
    try {
      let query = `SELECT
-        CAST(AVG(height) as INT) as avg_height,
+        CAST(AVG(height) as INT) as avgHeight,
        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(size) as INT) as avg_size
+        CAST(AVG(size) as INT) as avgSize
      FROM blocks`;

      if (interval !== null) {
@@ -581,9 +594,9 @@ class BlocksRepository {
   public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
    try {
      let query = `SELECT
-        CAST(AVG(height) as INT) as avg_height,
+        CAST(AVG(height) as INT) as avgHeight,
        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(weight) as INT) as avg_weight
+        CAST(AVG(weight) as INT) as avgWeight
      FROM blocks`;

      if (interval !== null) {
--- a/backend/src/routes.ts
+++ b/backend/src/routes.ts
@@ -619,6 +619,14 @@ class Routes {
  }

  public async $getHistoricalHashrate(req: Request, res: Response) {
+    let currentHashrate = 0, currentDifficulty = 0;
+    try {
+      currentHashrate = await bitcoinClient.getNetworkHashPs();
+      currentDifficulty = await bitcoinClient.getDifficulty();
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
+    }
+
    try {
      const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
      const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
@@ -630,8 +638,8 @@ class Routes {
      res.json({
        hashrates: hashrates,
        difficulty: difficulty,
-        currentHashrate: await bitcoinClient.getNetworkHashPs(),
-        currentDifficulty: await bitcoinClient.getDifficulty(),
+        currentHashrate: currentHashrate,
+        currentDifficulty: currentDifficulty,
      });
    } catch (e) {
      res.status(500).send(e instanceof Error ? e.message : e);
@@ -669,14 +677,12 @@ class Routes {
  public async $getHistoricalBlockFeeRates(req: Request, res: Response) {
    try {
      const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval);
-      const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
+      const blockCount = await BlocksRepository.$blockCount(null, null);
      res.header('Pragma', 'public');
      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json({
-        oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
-        blockFeeRates: blockFeeRates,
-      });
+      res.json(blockFeeRates);
    } catch (e) {
      res.status(500).send(e instanceof Error ? e.message : e);
    }
@@ -720,20 +726,22 @@ class Routes {
    }
  }

-  public async getBlocksExtras(req: Request, res: Response) {
+  public async getBlocks(req: Request, res: Response) {
    try {
-      const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(await blocks.$getBlocksExtras(height, 15));
+      if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+        const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+        res.json(await blocks.$getBlocks(height, 15));
+      } else { // Liquid, Bisq
+        return await this.getLegacyBlocks(req, res);
+      }
    } catch (e) {
      res.status(500).send(e instanceof Error ? e.message : e);
    }
  }

-  public async getBlocks(req: Request, res: Response) {
+  public async getLegacyBlocks(req: Request, res: Response) {
    try {
-      loadingIndicators.setProgress('blocks', 0);
-
      const returnBlocks: IEsploraApi.Block[] = [];
      const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();

@@ -757,16 +765,15 @@ class Routes {
          returnBlocks.push(block);
          nextHash = block.previousblockhash;
        }
-        loadingIndicators.setProgress('blocks', i / 10 * 100);
      }

+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
      res.json(returnBlocks);
    } catch (e) {
-      loadingIndicators.setProgress('blocks', 100);
      res.status(500).send(e instanceof Error ? e.message : e);
    }
  }
-
+  
  public async getBlockTransactions(req: Request, res: Response) {
    try {
      loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
--- a/backend/src/tasks/pools-updater.ts
+++ b/backend/src/tasks/pools-updater.ts
@@ -16,7 +16,7 @@ class PoolsUpdater {
  }

  public async updatePoolsJson() {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || config.DATABASE.ENABLED === false) {
      return;
    }

--- a/backend/src/utils/blocks-utils.ts
+++ b/backend/src/utils/blocks-utils.ts
@@ -15,7 +15,7 @@ export function prepareBlock(block: any): BlockExtended {
    weight: block.weight,
    previousblockhash: block.previousblockhash,
    extras: {
-      coinbaseRaw: block.coinbase_raw ?? block.extras.coinbaseRaw,
+      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
      feeRange: block.feeRange ?? block.fee_span,
      reward: block.reward ?? block?.extras?.reward,