Merge branch 'master' into natsoni/federation-utxos-expiry

backend/src/__fixtures__/mempool-config.template.json

@@ -1,6 +1,7 @@
 {
   "MEMPOOL": {
     "ENABLED": true,
+    "OFFICIAL": false,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
     "BLOCKS_SUMMARIES_INDEXING": true,
@@ -79,7 +80,8 @@
     "USERNAME": "__DATABASE_USERNAME__",
     "PASSWORD": "__DATABASE_PASSWORD__",
     "PID_DIR": "__DATABASE_PID_FILE__",
-    "TIMEOUT": 3000
+    "TIMEOUT": 3000,
+    "POOL_SIZE": 100
   },
   "SYSLOG": {
     "ENABLED": false,

backend/src/__tests__/config.test.ts

@@ -14,6 +14,7 @@ describe('Mempool Backend Config', () => {
 
       expect(config.MEMPOOL).toStrictEqual({
         ENABLED: true,
+        OFFICIAL: false,
         NETWORK: 'mainnet',
         BACKEND: 'none',
         BLOCKS_SUMMARIES_INDEXING: false,
@@ -93,7 +94,8 @@ describe('Mempool Backend Config', () => {
         USERNAME: 'mempool',
         PASSWORD: 'mempool',
         TIMEOUT: 180000,
-        PID_DIR: ''
+        PID_DIR: '',
+        POOL_SIZE: 100,
       });
 
       expect(config.SYSLOG).toStrictEqual({

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -29,6 +29,7 @@ export interface AbstractBitcoinApi {
   $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;
 
   startHealthChecks(): void;
+  getHealthStatus(): HealthCheckHost[];
 }
 export interface BitcoinRpcCredentials {
   host: string;
@@ -38,3 +39,15 @@ export interface BitcoinRpcCredentials {
   timeout: number;
   cookie?: string;
 }
+
+export interface HealthCheckHost {
+  host: string;
+  active: boolean;
+  rtt: number;
+  latestHeight: number;
+  socket: boolean;
+  outOfSync: boolean;
+  unreachable: boolean;
+  checked: boolean;
+  lastChecked: number;
+}

backend/src/api/bitcoin/bitcoin-api.ts

@@ -1,5 +1,5 @@
 import * as bitcoinjs from 'bitcoinjs-lib';
-import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
 import { IBitcoinApi } from './bitcoin-api.interface';
 import { IEsploraApi } from './esplora-api.interface';
 import blocks from '../blocks';
@@ -382,6 +382,10 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   public startHealthChecks(): void {};
+
+  public getHealthStatus() {
+    return [];
+  }
 }
 
 export default BitcoinApi;

backend/src/api/bitcoin/esplora-api.ts

@@ -1,7 +1,7 @@
 import config from '../../config';
-import axios, { AxiosResponse } from 'axios';
+import axios, { AxiosResponse, isAxiosError } from 'axios';
 import http from 'http';
-import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import logger from '../../logger';
 import { Common } from '../common';
@@ -10,6 +10,7 @@ interface FailoverHost {
   host: string,
   rtts: number[],
   rtt: number,
+  timedOut?: boolean,
   failures: number,
   latestHeight?: number,
   socket?: boolean,
@@ -17,6 +18,7 @@ interface FailoverHost {
   unreachable?: boolean,
   preferred?: boolean,
   checked: boolean,
+  lastChecked?: number,
 }
 
 class FailoverRouter {
@@ -108,14 +110,20 @@ class FailoverRouter {
           host.rtts = [];
           host.rtt = Infinity;
         }
+        host.timedOut = false;
       } catch (e) {
         host.outOfSync = true;
         host.unreachable = true;
         host.rtts = [];
         host.rtt = Infinity;
+        if (isAxiosError(e) && (e.code === 'ECONNABORTED' || e.code === 'ETIMEDOUT')) {
+          host.timedOut = true;
+        } else {
+          host.timedOut = false;
+        }
       }
       host.checked = true;
-      
+      host.lastChecked = Date.now();
 
       // switch if the current host is out of sync or significantly slower than the next best alternative
       const rankOrder = this.sortHosts();
@@ -143,7 +151,7 @@ class FailoverRouter {
 
   private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string {
     const heightStatus = !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅'));
-    return `${host === active ? '⭐️' : '  '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : '    -  '} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : '  '}`;
+    return `${host === active ? '⭐️' : '  '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : (host.timedOut ? '  ⌛️💥 ' : '    -  ')} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : '  '}`;
   }
 
   private updateFallback(): FailoverHost[] {
@@ -157,7 +165,7 @@ class FailoverRouter {
   }
 
   // sort hosts by connection quality, and update default fallback
-  private sortHosts(): FailoverHost[] {
+  public sortHosts(): FailoverHost[] {
     // sort by connection quality
     return this.hosts.slice().sort((a, b) => {
       if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) {
@@ -342,6 +350,24 @@ class ElectrsApi implements AbstractBitcoinApi {
   public startHealthChecks(): void {
     this.failoverRouter.startHealthChecks();
   }
+
+  public getHealthStatus(): HealthCheckHost[] {
+    if (config.MEMPOOL.OFFICIAL) {
+      return this.failoverRouter.sortHosts().map(host => ({
+        host: host.host,
+        active: host === this.failoverRouter.activeHost,
+        rtt: host.rtt,
+        latestHeight: host.latestHeight || 0,
+        socket: !!host.socket,
+        outOfSync: !!host.outOfSync,
+        unreachable: !!host.unreachable,
+        checked: !!host.checked,
+        lastChecked: host.lastChecked || 0,
+      }));
+    } else {
+      return [];
+    }
+  }
 }
 
 export default ElectrsApi;

backend/src/api/websocket-handler.ts

@@ -26,6 +26,7 @@ import mempool from './mempool';
 import statistics from './statistics/statistics';
 import accelerationCosts from './acceleration';
 import accelerationRepository from '../repositories/AccelerationRepository';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
 
 interface AddressTransactions {
   mempool: MempoolTransactionExtended[],
@@ -39,6 +40,7 @@ const wantable = [
   'mempool-blocks',
   'live-2h-chart',
   'stats',
+  'tomahawk',
 ];
 
 class WebsocketHandler {
@@ -123,7 +125,7 @@ class WebsocketHandler {
             for (const sub of wantable) {
               const key = `want-${sub}`;
               const wants = parsedMessage.data.includes(sub);
-              if (wants && client['wants'] && !client[key]) {
+              if (wants && !client[key]) {
                 wantNow[key] = true;
               }
               client[key] = wants;
@@ -147,6 +149,10 @@ class WebsocketHandler {
             response['da'] = this.socketData['da'];
           }
 
+          if (wantNow['want-tomahawk']) {
+            response['tomahawk'] = JSON.stringify(bitcoinApi.getHealthStatus());
+          }
+
           if (parsedMessage && parsedMessage['track-tx']) {
             if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
               client['track-tx'] = parsedMessage['track-tx'];
@@ -546,6 +552,10 @@ class WebsocketHandler {
         response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
       }
 
+      if (client['want-tomahawk']) {
+        response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus());
+      }
+
       if (client['track-mempool-tx']) {
         const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']);
         if (tx) {
@@ -909,6 +919,10 @@ class WebsocketHandler {
         response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
       }
 
+      if (client['want-tomahawk']) {
+        response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus());
+      }
+
       if (client['track-tx']) {
         const trackTxid = client['track-tx'];
         if (trackTxid && confirmedTxids[trackTxid]) {

backend/src/config.ts

@@ -5,6 +5,7 @@ const configFromFile = require(
 interface IConfig {
   MEMPOOL: {
     ENABLED: boolean;
+    OFFICIAL: boolean;
     NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
@@ -103,6 +104,7 @@ interface IConfig {
     PASSWORD: string;
     TIMEOUT: number;
     PID_DIR: string;
+    POOL_SIZE: number;
   };
   SYSLOG: {
     ENABLED: boolean;
@@ -161,6 +163,7 @@ interface IConfig {
 const defaults: IConfig = {
   'MEMPOOL': {
     'ENABLED': true,
+    'OFFICIAL': false,
     'NETWORK': 'mainnet',
     'BACKEND': 'none',
     'HTTP_PORT': 8999,
@@ -240,6 +243,7 @@ const defaults: IConfig = {
     'PASSWORD': 'mempool',
     'TIMEOUT': 180000,
     'PID_DIR': '',
+    'POOL_SIZE': 100,
   },
   'SYSLOG': {
     'ENABLED': true,

backend/src/database.ts

@@ -21,7 +21,7 @@ import { execSync } from 'child_process';
     database: config.DATABASE.DATABASE,
     user: config.DATABASE.USERNAME,
     password: config.DATABASE.PASSWORD,
-    connectionLimit: 10,
+    connectionLimit: config.DATABASE.POOL_SIZE,
     supportBigNumbers: true,
     timezone: '+00:00',
   };

backend/src/index.ts

@@ -155,11 +155,17 @@ class Server {
     }
 
     if (Common.isLiquid()) {
-      try {
-        icons.loadIcons();
-      } catch (e) {
-        logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
-      }
+      const refreshIcons = () => {
+        try {
+          icons.loadIcons();
+        } catch (e) {
+          logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
+        }
+      };
+      // Run once on startup.
+      refreshIcons();
+      // Matches crontab refresh interval for asset db.
+      setInterval(refreshIcons, 3600_000);
     }
 
     priceUpdater.$run();