Merge branch 'master' into nymkappa/bugfix/pool-parser-db-check

backend/README.md

@@ -1,22 +1,161 @@
-# Setup backend watchers
+# Mempool Backend
 
-The backend is static. Typescript scripts are compiled into the `dist` folder and served through a node web server.
+These instructions are mostly intended for developers, but can be used as a basis for personal or small-scale production setups. 
 
-You can avoid the manual shutdown/recompile/restart command line cycle by using a watcher.
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
 
-Make sure you are in the `backend` directory `cd backend`.
+See other ways to set up Mempool on [the main README](/../../#installation-methods).
 
-1. Install nodemon and ts-node
+Jump to a section in this doc:
+- [Set Up the Backend](#setup)
+- [Development Tips](#development-tips)
+
+## Setup
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
 
 ```
-sudo npm install -g ts-node nodemon
+git clone https://github.com/mempool/mempool
+cd mempool
 ```
 
-2. Run the watcher
+Check out the latest release:
 
-> Note: You can find your npm global binary folder using `npm -g bin`, where nodemon will be installed.
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Configure Bitcoin Core
+
+Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
+
+```
+txindex=1
+server=1
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+### 3. Configure Electrum Server
+
+[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
+
+**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
+
+### 4. Configure MariaDB
+
+_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
+
+Get MariaDB from your operating system's package manager:
+
+```
+# Debian, Ubuntu, etc.
+apt-get install mariadb-server mariadb-client
+
+# macOS
+brew install mariadb
+mysql.server start
+```
+
+Create a database and grant privileges:
+
+```
+MariaDB [(none)]> drop database mempool;
+Query OK, 0 rows affected (0.00 sec)
+
+MariaDB [(none)]> create database mempool;
+Query OK, 1 row affected (0.00 sec)
+
+MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+Query OK, 0 rows affected (0.00 sec)
+```
+
+### 5. Prepare Mempool Backend
+
+#### Build
+
+_Node.js 16 and npm 7 are recommended._
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd backend
+npm install # add --prod for production
+npm run build
+```
+
+#### Configure
+
+In the backend folder, make a copy of the sample config file:
+
+```
+cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` as needed. 
+
+In particular, make sure:
+- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
+- the correct `BACKEND` is specified in `MEMPOOL`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
+  - "none" if you're not using any Electrum Server
+
+### 6. Run Mempool Backend
+
+Run the Mempool backend:
+
+```
+npm run start
+```
+
+When it's running, you should see output like this:
+
+```
+Mempool updated in 0.189 seconds
+Updating mempool
+Mempool updated in 0.096 seconds
+Updating mempool
+Mempool updated in 0.099 seconds
+Updating mempool
+Calculated fee for transaction 1 / 10
+Calculated fee for transaction 2 / 10
+Calculated fee for transaction 3 / 10
+Calculated fee for transaction 4 / 10
+Calculated fee for transaction 5 / 10
+Calculated fee for transaction 6 / 10
+Calculated fee for transaction 7 / 10
+Calculated fee for transaction 8 / 10
+Calculated fee for transaction 9 / 10
+Calculated fee for transaction 10 / 10
+Mempool updated in 0.243 seconds
+Updating mempool
+```
+
+### 7. Set Up Mempool Frontend
+With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
+
+## Development Tips
+
+### Set Up Backend Watchers
+
+The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
+
+As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
+
+First, install `nodemon` and `ts-node`:
+
+```
+npm install -g ts-node nodemon
+```
+
+Then, run the watcher:
 
 ```
 nodemon src/index.ts --ignore cache/ --ignore pools.json
 ```
 
+`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.

backend/src/api/blocks.ts

@@ -426,25 +426,32 @@ class Blocks {
         return returnBlocks;
       }
 
+      if (currentHeight === 0 && Common.indexingEnabled()) {
+        currentHeight = await blocksRepository.$mostRecentBlockHeight();
+      }
+
       // Check if block height exist in local cache to skip the hash lookup
       const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
       let startFromHash: string | null = null;
       if (blockByHeight) {
         startFromHash = blockByHeight.id;
-      } else {
+      } else if (!Common.indexingEnabled()) {
         startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
       }
 
       let nextHash = startFromHash;
       for (let i = 0; i < limit && currentHeight >= 0; i++) {
         let block = this.getBlocks().find((b) => b.height === currentHeight);
-        if (!block && Common.indexingEnabled()) {
+        if (block) {
+          returnBlocks.push(block);
+        } else if (Common.indexingEnabled()) {
           block = await this.$indexBlock(currentHeight);
-        } else if (!block) {
+          returnBlocks.push(block);
+        } else if (nextHash != null) {
           block = prepareBlock(await bitcoinApi.$getBlock(nextHash));
+          nextHash = block.previousblockhash;
+          returnBlocks.push(block);
         }
-        returnBlocks.push(block);
-        nextHash = block.previousblockhash;
         currentHeight--;
       }
 

backend/src/api/mining.ts

@@ -94,8 +94,13 @@ class Mining {
     poolsStatistics['blockCount'] = blockCount;
 
     const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
-    const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
-    poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
+
+    try {
+      poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      poolsStatistics['lastEstimatedHashrate'] = 0;
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }
 
     return poolsStatistics;
   }
@@ -118,7 +123,12 @@ class Mining {
     const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
     const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
 
-    const currentEstimatedkHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    let currentEstimatedHashrate = 0;
+    try {
+      currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }
 
     return {
       pool: pool,
@@ -132,7 +142,7 @@ class Mining {
         '24h': blockCount24h / totalBlock24h,
         '1w': blockCount1w / totalBlock1w,
       },
-      estimatedHashrate: currentEstimatedkHashrate * (blockCount24h / totalBlock24h),
+      estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
       reportedHashrate: null,
     };
   }

backend/src/repositories/BlocksRepository.ts

@@ -121,6 +121,19 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Return most recent block height
+   */
+  public async $mostRecentBlockHeight(): Promise<number> {
+    try {
+      const [row] = await DB.query('SELECT MAX(height) as maxHeight from blocks');
+      return row[0]['maxHeight'];
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get blocks count for a period
    */

backend/src/routes.ts

@@ -619,6 +619,14 @@ class Routes {
   }
 
   public async $getHistoricalHashrate(req: Request, res: Response) {
+    let currentHashrate = 0, currentDifficulty = 0;
+    try {
+      currentHashrate = await bitcoinClient.getNetworkHashPs();
+      currentDifficulty = await bitcoinClient.getDifficulty();
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
+    }
+
     try {
       const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
       const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
@@ -630,8 +638,8 @@ class Routes {
       res.json({
         hashrates: hashrates,
         difficulty: difficulty,
-        currentHashrate: await bitcoinClient.getNetworkHashPs(),
-        currentDifficulty: await bitcoinClient.getDifficulty(),
+        currentHashrate: currentHashrate,
+        currentDifficulty: currentDifficulty,
       });
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);