attempt to recover from stalling the main loop

Backend block tip height endpoint

backend/mempool-config.sample.json

@@ -43,7 +43,9 @@
     "TLS_ENABLED": true
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
+    "REST_API_URL": "http://127.0.0.1:3000",
+    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
+    "RETRY_UNIX_SOCKET_AFTER": 30000
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.5.0",
+  "version": "2.6.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -34,35 +34,35 @@
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@babel/core": "^7.20.12",
+    "@babel/core": "^7.21.3",
     "@mempool/electrum-client": "1.1.9",
-    "@types/node": "^16.18.11",
+    "@types/node": "^18.15.3",
     "axios": "~0.27.2",
     "bitcoinjs-lib": "~6.1.0",
     "crypto-js": "~4.1.1",
     "express": "~4.18.2",
     "maxmind": "~4.3.8",
-    "mysql2": "~2.3.3",
+    "mysql2": "~3.2.0",
     "node-worker-threads-pool": "~1.5.1",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.7.4",
-    "ws": "~8.11.0"
+    "ws": "~8.13.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.20.7",
+    "@babel/core": "^7.21.3",
     "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.15",
-    "@types/jest": "^29.2.5",
+    "@types/jest": "^29.5.0",
     "@types/ws": "~8.5.4",
-    "@typescript-eslint/eslint-plugin": "^5.48.1",
-    "@typescript-eslint/parser": "^5.48.1",
-    "eslint": "^8.31.0",
-    "eslint-config-prettier": "^8.5.0",
-    "jest": "^29.3.1",
-    "prettier": "^2.8.2",
-    "ts-jest": "^29.0.3",
+    "@typescript-eslint/eslint-plugin": "^5.55.0",
+    "@typescript-eslint/parser": "^5.55.0",
+    "eslint": "^8.36.0",
+    "eslint-config-prettier": "^8.7.0",
+    "jest": "^29.5.0",
+    "prettier": "^2.8.4",
+    "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -44,7 +44,9 @@
     "TLS_ENABLED": true
   },
   "ESPLORA": {
-    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+    "REST_API_URL": "__ESPLORA_REST_API_URL__",
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
+    "RETRY_UNIX_SOCKET_AFTER": "__ESPLORA_RETRY_UNIX_SOCKET_AFTER__"
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => {
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
 
-      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, RETRY_UNIX_SOCKET_AFTER: 30000 });
 
       expect(config.CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -94,6 +94,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
@@ -110,7 +111,6 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
-          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
@@ -589,10 +589,14 @@ class BitcoinRoutes {
     }
   }
 
-  private async getBlockTipHeight(req: Request, res: Response) {
+  private getBlockTipHeight(req: Request, res: Response) {
     try {
-      const result = await bitcoinApi.$getBlockHeightTip();
-      res.json(result);
+      const result = blocks.getCurrentBlockHeight();
+      if (!result) {
+        return res.status(503).send(`Service Temporarily Unavailable`);
+      }
+      res.setHeader('content-type', 'text/plain');
+      res.send(result.toString());
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/bitcoin/esplora-api.ts

@@ -5,11 +5,14 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 
 const axiosConnection = axios.create({
-  httpAgent: new http.Agent({ keepAlive: true })
+  httpAgent: new http.Agent({ keepAlive: true, })
 });
 
 class ElectrsApi implements AbstractBitcoinApi {
-  axiosConfig: AxiosRequestConfig = {
+  axiosConfig: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
+    socketPath: config.ESPLORA.UNIX_SOCKET_PATH,
+    timeout: 10000,
+  } : {
     timeout: 10000,
   };
 

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -36,6 +36,8 @@ class Blocks {
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
   private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
+  private mainLoopTimeout: number = 120000;
+
   constructor() { }
 
   public getBlocks(): BlockExtended[] {
@@ -200,8 +202,15 @@ class Blocks {
       extras.segwitTotalWeight = 0;
     } else {
       const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
-      extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      let feeStats = {
+        medianFee: stats.feerate_percentiles[2], // 50th percentiles
+        feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
+      };
+      if (transactions?.length > 1) {
+        feeStats = Common.calcEffectiveFeeStatistics(transactions);
+      }
+      extras.medianFee = feeStats.medianFee;
+      extras.feeRange = feeStats.feeRange;
       extras.totalFees = stats.totalfee;
       extras.avgFee = stats.avgfee;
       extras.avgFeeRate = stats.avgfeerate;
@@ -403,12 +412,13 @@ class Blocks {
     try {
       // Get all indexed block hash
       const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
-      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
 
       if (!unindexedBlockHeights?.length) {
         return;
       }
 
+      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
+
       // Logging
       let count = 0;
       let countThisRun = 0;
@@ -520,8 +530,12 @@ class Blocks {
   }
 
   public async $updateBlocks() {
+    // throw an error if this stalls the main loop for more than 2 minutes
+    const timer = this.startTimer();
+
     let fastForwarded = false;
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
+    this.updateTimerProgress(timer, 'got block height tip');
 
     if (this.blocks.length === 0) {
       this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1);
@@ -539,16 +553,21 @@ class Blocks {
 
     if (!this.lastDifficultyAdjustmentTime) {
       const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      this.updateTimerProgress(timer, 'got blockchain info for initial difficulty adjustment');
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
+        this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment');
         const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+        this.updateTimerProgress(timer, 'got block for initial difficulty adjustment');
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
+          this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
           const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
+          this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -563,52 +582,66 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
         await chainTips.updateOrphanedBlocks();
       }
 
+      this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
       const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
       const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
-      const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+      const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
+      const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
+      this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
       // start async callbacks
+      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
       const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
 
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
+          this.updateTimerProgress(timer, `got block by height for ${this.currentBlockHeight}`);
           if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
-            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
+            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
             // We assume there won't be a reorg with more than 10 block depth
+            this.updateTimerProgress(timer, `rolling back diverged chain from ${this.currentBlockHeight}`);
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
+            this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
+              this.updateTimerProgress(timer, `reindexed block`);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
+              this.updateTimerProgress(timer, `reindexed block summary`);
               if (config.MEMPOOL.CPFP_INDEXING) {
                 await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+                this.updateTimerProgress(timer, `reindexed block cpfp`);
               }
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
-            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
             indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
+          this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`);
 
           const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
           if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
             await blocksRepository.$saveBlockPrices([{
               height: blockExtended.height,
               priceId: lastestPriceId,
             }]);
+            this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
           } else {
-            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+            logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
             setTimeout(() => {
               indexer.runSingleTask('blocksPrices');
             }, 10000);
@@ -617,9 +650,11 @@ class Blocks {
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
+            this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
           }
           if (config.MEMPOOL.CPFP_INDEXING) {
-            this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
+            this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
+            this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
           }
         }
       }
@@ -632,6 +667,7 @@ class Blocks {
             difficulty: block.difficulty,
             adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
           });
+          this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
         }
 
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
@@ -656,7 +692,38 @@ class Blocks {
       }
 
       // wait for pending async callbacks to finish
+      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
       await Promise.all(callbackPromises);
+      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
+    }
+
+    this.clearTimer(timer);
+  }
+
+  private startTimer() {
+    const state: any = {
+      start: Date.now(),
+      progress: 'begin $updateBlocks',
+      timer: null,
+    };
+    state.timer = setTimeout(() => {
+      logger.err(`$updateBlocks stalled at "${state.progress}`);
+    }, this.mainLoopTimeout);
+    return state;
+  }
+
+  private updateTimerProgress(state, msg) {
+    state.progress = msg;
+    if (Date.now() - state.start > this.mainLoopTimeout) {
+      // abort the function if it already timed out
+      logger.err(`$updateBlocks attempted to resume after "${state.progress}"`);
+      throw new Error(`$updateBlocks attempted to resume after "${state.progress}"`);
+    }
+  }
+
+  private clearTimer(state) {
+    if (state.timer) {
+      clearTimeout(state.timer);
     }
   }
 
@@ -728,7 +795,7 @@ class Blocks {
 
     // Index the response if needed
     if (Common.blocksSummariesIndexingEnabled() === true) {
-      await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary});
+      await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions);
     }
 
     return summary.transactions;
@@ -844,11 +911,12 @@ class Blocks {
         if (cleanBlock.fee_amt_percentiles === null) {
           const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
           const summary = this.summarizeBlock(block);
-          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
           cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
         }
         if (cleanBlock.fee_amt_percentiles !== null) {
           cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+          await blocksRepository.$updateFeeAmounts(cleanBlock.hash, cleanBlock.fee_amt_percentiles, cleanBlock.median_fee_amt);
         }
       }
 
@@ -913,42 +981,20 @@ class Blocks {
   public async $indexCPFP(hash: string, height: number): Promise<void> {
     const block = await bitcoinClient.getBlock(hash, 2);
     const transactions = block.tx.map(tx => {
-      tx.vsize = tx.weight / 4;
       tx.fee *= 100_000_000;
       return tx;
     });
 
-    const clusters: any[] = [];
+    const summary = Common.calculateCpfp(height, transactions);
 
-    let cluster: TransactionStripped[] = [];
-    let ancestors: { [txid: string]: boolean } = {};
-    for (let i = transactions.length - 1; i >= 0; i--) {
-      const tx = transactions[i];
-      if (!ancestors[tx.txid]) {
-        let totalFee = 0;
-        let totalVSize = 0;
-        cluster.forEach(tx => {
-          totalFee += tx?.fee || 0;
-          totalVSize += tx.vsize;
-        });
-        const effectiveFeePerVsize = totalFee / totalVSize;
-        if (cluster.length > 1) {
-          clusters.push({
-            root: cluster[0].txid,
-            height,
-            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }),
-            effectiveFeePerVsize,
-          });
-        }
-        cluster = [];
-        ancestors = {};
-      }
-      cluster.push(tx);
-      tx.vin.forEach(vin => {
-        ancestors[vin.txid] = true;
-      });
-    }
-    const result = await cpfpRepository.$batchSaveClusters(clusters);
+    await this.$saveCpfp(hash, height, summary);
+
+    const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
+    await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
+  }
+
+  public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {
+    const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
     if (!result) {
       await cpfpRepository.$insertProgressMarker(height);
     }

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';
@@ -345,4 +345,99 @@ export class Common {
       };
     }
   }
+
+  static calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary {
+    const clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[] = [];
+    let cluster: TransactionExtended[] = [];
+    let ancestors: { [txid: string]: boolean } = {};
+    const txMap = {};
+    for (let i = transactions.length - 1; i >= 0; i--) {
+      const tx = transactions[i];
+      txMap[tx.txid] = tx;
+      if (!ancestors[tx.txid]) {
+        let totalFee = 0;
+        let totalVSize = 0;
+        cluster.forEach(tx => {
+          totalFee += tx?.fee || 0;
+          totalVSize += (tx.weight / 4);
+        });
+        const effectiveFeePerVsize = totalFee / totalVSize;
+        if (cluster.length > 1) {
+          clusters.push({
+            root: cluster[0].txid,
+            height,
+            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }),
+            effectiveFeePerVsize,
+          });
+        }
+        cluster.forEach(tx => {
+          txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        });
+        cluster = [];
+        ancestors = {};
+      }
+      cluster.push(tx);
+      tx.vin.forEach(vin => {
+        ancestors[vin.txid] = true;
+      });
+    }
+    return {
+      transactions,
+      clusters,
+    };
+  }
+
+  static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats {
+    const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
+
+    let weightCount = 0;
+    let medianFee = 0;
+    let medianWeight = 0;
+
+    // calculate the "medianFee" as the average fee rate of the middle 10000 weight units of transactions
+    const leftBound = 1995000;
+    const rightBound = 2005000;
+    for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) {
+      const left = weightCount;
+      const right = weightCount + sortedTxs[i].weight;
+      if (right > leftBound) {
+        const weight = Math.min(right, rightBound) - Math.max(left, leftBound);
+        medianFee += (sortedTxs[i].rate * (weight / 4) );
+        medianWeight += weight;
+      }
+      weightCount += sortedTxs[i].weight;
+    }
+    const medianFeeRate = medianWeight ? (medianFee / (medianWeight / 4)) : 0;
+
+    // minimum effective fee heuristic:
+    // lowest of
+    // a) the 1st percentile of effective fee rates
+    // b) the minimum effective fee rate in the last 2% of transactions (in block order)
+    const minFee = Math.min(
+      Common.getNthPercentile(1, sortedTxs).rate,
+      transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity)
+    );
+
+    // maximum effective fee heuristic:
+    // highest of
+    // a) the 99th percentile of effective fee rates
+    // b) the maximum effective fee rate in the first 2% of transactions (in block order)
+    const maxFee = Math.max(
+      Common.getNthPercentile(99, sortedTxs).rate,
+      transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0)
+    );
+
+    return {
+      medianFee: medianFeeRate,
+      feeRange: [
+        minFee,
+        [10,25,50,75,90].map(n => Common.getNthPercentile(n, sortedTxs).rate),
+        maxFee,
+      ].flat(),
+    };
+  }
+
+  static getNthPercentile(n: number, sortedDistribution: any[]): any {
+    return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
+  }
 }

backend/src/api/disk-cache.ts

@@ -164,7 +164,7 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+          logger.err('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
         }
       }
 

backend/src/api/liquid/icons.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import logger from '../../logger';
 
 class Icons {
-  private static FILE_NAME = './icons.json';
+  private static FILE_NAME = '/elements/asset_registry_db/icons.json';
   private iconIds: string[] = [];
   private icons: { [assetId: string]: string; } = {};
 

backend/src/api/mempool-blocks.ts

@@ -59,7 +59,7 @@ class MempoolBlocks {
     // Loop through and traverse all ancestors and sum up all the sizes + fees
     // Pass down size + fee to all unconfirmed children
     let sizes = 0;
-    memPoolArray.forEach((tx, i) => {
+    memPoolArray.forEach((tx) => {
       sizes += tx.weight;
       if (sizes > 4000000 * 8) {
         return;
@@ -74,7 +74,7 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const blocks = this.calculateMempoolBlocks(memPoolArray);
 
     if (saveResults) {
       const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
@@ -85,26 +85,23 @@ class MempoolBlocks {
     return blocks;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
     let blockWeight = 0;
-    let blockSize = 0;
     let transactions: TransactionExtended[] = [];
     transactionsSorted.forEach((tx) => {
       if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
         || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
         blockWeight += tx.weight;
-        blockSize += tx.size;
         transactions.push(tx);
       } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions));
         blockWeight = tx.weight;
-        blockSize = tx.size;
         transactions = [tx];
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions));
     }
 
     return mempoolBlocks;
@@ -298,10 +295,10 @@ class MempoolBlocks {
     });
 
     // unpack the condensed blocks into proper mempool blocks
-    const mempoolBlocks = blocks.map((transactions, blockIndex) => {
+    const mempoolBlocks = blocks.map((transactions) => {
       return this.dataToMempoolBlocks(transactions.map(tx => {
         return mempool[tx.txid] || null;
-      }).filter(tx => !!tx), blockIndex);
+      }).filter(tx => !!tx));
     });
 
     if (saveResults) {
@@ -313,7 +310,7 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
+  private dataToMempoolBlocks(transactions: TransactionExtended[]): MempoolBlockWithTransactions {
     let totalSize = 0;
     let totalWeight = 0;
     const fitTransactions: TransactionExtended[] = [];
@@ -324,22 +321,14 @@ class MempoolBlocks {
         fitTransactions.push(tx);
       }
     });
-    let rangeLength = 4;
-    if (blocksIndex === 0) {
-      rangeLength = 8;
-    }
-    if (transactions.length > 4000) {
-      rangeLength = 6;
-    } else if (transactions.length > 10000) {
-      rangeLength = 8;
-    }
+    const feeStats = Common.calcEffectiveFeeStatistics(transactions);
     return {
       blockSize: totalSize,
       blockVSize: totalWeight / 4,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
-      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
-      feeRange: Common.getFeesInRange(transactions, rangeLength),
+      medianFee: feeStats.medianFee, // Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+      feeRange: feeStats.feeRange, //Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
       transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
     };

backend/src/api/mempool.ts

@@ -36,6 +36,8 @@ class Mempool {
   private timer = new Date().getTime();
   private missingTxCount = 0;
 
+  private mainLoopTimeout: number = 120000;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
   }
@@ -119,10 +121,15 @@ class Mempool {
 
   public async $updateMempool(): Promise<void> {
     logger.debug(`Updating mempool...`);
+
+    // throw an error if this stalls the main loop for more than 2 minutes
+    const timer = this.startTimer();
+
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
     const transactions = await bitcoinApi.$getRawMempool();
+    this.updateTimerProgress(timer, 'got raw mempool');
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
@@ -146,6 +153,7 @@ class Mempool {
       if (!this.mempoolCache[txid]) {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
+          this.updateTimerProgress(timer, 'fetched new transaction');
           this.mempoolCache[txid] = transaction;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
@@ -223,12 +231,43 @@ class Mempool {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
     if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      this.updateTimerProgress(timer, 'running async mempool callback');
       await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+      this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
     const end = new Date().getTime();
     const time = end - start;
     logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
+
+    this.clearTimer(timer);
+  }
+
+  private startTimer() {
+    const state: any = {
+      start: Date.now(),
+      progress: 'begin $updateMempool',
+      timer: null,
+    };
+    state.timer = setTimeout(() => {
+      logger.err(`$updateMempool stalled at "${state.progress}`);
+    }, this.mainLoopTimeout);
+    return state;
+  }
+
+  private updateTimerProgress(state, msg) {
+    state.progress = msg;
+    if (Date.now() - state.start > this.mainLoopTimeout) {
+      // abort the function if it already timed out
+      logger.err(`$updateMempool attempted to resume after "${state.progress}"`);
+      throw new Error(`$updateMempool attempted to resume after "${state.progress}"`);
+    }
+  }
+
+  private clearTimer(state) {
+    if (state.timer) {
+      clearTimeout(state.timer);
+    }
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {

backend/src/api/mining/mining.ts

@@ -452,7 +452,7 @@ class Mining {
       const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
       if (elapsedSeconds > 5) {
         const progress = Math.round(totalBlockChecked / blocks.length * 100);
-        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
+        logger.debug(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
         timer = new Date().getTime() / 1000;
       }
     }
@@ -558,8 +558,10 @@ class Mining {
       currentBlockHeight -= 10000;
     }
 
-    if (totalIndexed) {
-      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    if (totalIndexed > 0) {
+      logger.info(`Indexing missing coinstatsindex data completed. Indexed ${totalIndexed}`, logger.tags.mining);
+    } else {
+      logger.debug(`Indexing missing coinstatsindex data completed. Indexed 0.`, logger.tags.mining);
     }
   }
 

backend/src/api/tx-selection-worker.ts

@@ -107,7 +107,7 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
 
     if (nextTx && !nextTx?.used) {
       // Check if the package fits into this block
-      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+      if (blocks.length >= 7 || (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS)) {
         const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
         // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
         const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
@@ -175,34 +175,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
       overflow = [];
     }
   }
-  // pack any leftover transactions into the last block
-  for (const tx of overflow) {
-    if (!tx || tx?.used) {
-      continue;
-    }
-    blockWeight += tx.weight;
-    const mempoolTx = mempool[tx.txid];
-    // update original copy of this tx with effective fee rate & relatives data
-    mempoolTx.effectiveFeePerVsize = tx.score;
-    if (tx.ancestorMap.size > 0) {
-      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
-      mempoolTx.cpfpRoot = tx.txid;
-    }
-    mempoolTx.cpfpChecked = true;
-    transactions.push(tx);
-    tx.used = true;
+
+  if (overflow.length > 0) {
+    logger.warn('GBT overflow list unexpectedly non-empty after final block constructed');
   }
-  const blockTransactions = transactions.map(t => mempool[t.txid]);
-  restOfArray.forEach(tx => {
-    blockWeight += tx.weight;
-    tx.effectiveFeePerVsize = tx.feePerVsize;
-    tx.cpfpChecked = false;
-    blockTransactions.push(tx);
-  });
-  if (blockTransactions.length) {
-    blocks.push(blockTransactions);
+  // add the final unbounded block if it contains any transactions
+  if (transactions.length > 0) {
+    blocks.push(transactions.map(t => mempool[t.txid]));
   }
-  transactions = [];
 
   const end = Date.now();
   const time = end - start;

backend/src/config.ts

@@ -37,6 +37,8 @@ interface IConfig {
   };
   ESPLORA: {
     REST_API_URL: string;
+    UNIX_SOCKET_PATH: string | void | null;
+    RETRY_UNIX_SOCKET_AFTER: number;
   };
   LIGHTNING: {
     ENABLED: boolean;
@@ -163,6 +165,8 @@ const defaults: IConfig = {
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
+    'UNIX_SOCKET_PATH': null,
+    'RETRY_UNIX_SOCKET_AFTER': 30000,
   },
   'ELECTRUM': {
     'HOST': '127.0.0.1',

backend/src/index.ts

@@ -179,8 +179,14 @@ class Server {
         }
       }
       memPool.deleteExpiredTransactions();
-      await blocks.$updateBlocks();
-      await memPool.$updateMempool();
+      await Promise.race([
+        blocks.$updateBlocks(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateBlocks timed out')), 180000))
+      ]);
+      await Promise.race([
+        memPool.$updateMempool(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateMempool timed out')), 180000))
+      ]);
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
@@ -276,7 +282,7 @@ class Server {
 
     if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
       this.warnedHeapCritical = true;
-      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
+      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit * 100).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
     }
     if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
       logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);

backend/src/mempool.interfaces.ts

@@ -214,6 +214,16 @@ export interface MempoolStats {
   tx_count: number;
 }
 
+export interface EffectiveFeeStats {
+  medianFee: number; // median effective fee rate
+  feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
+}
+
+export interface CpfpSummary {
+  transactions: TransactionExtended[];
+  clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];
+}
+
 export interface Statistic {
   id?: number;
   added: string;

backend/src/repositories/BlocksRepository.ts

@@ -1,4 +1,4 @@
-import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
@@ -13,6 +13,48 @@ import chainTips from '../api/chain-tips';
 import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
 
+interface DatabaseBlock {
+  id: string;
+  height: number;
+  version: number;
+  timestamp: number;
+  bits: number;
+  nonce: number;
+  difficulty: number;
+  merkle_root: string;
+  tx_count: number;
+  size: number;
+  weight: number;
+  previousblockhash: string;
+  mediantime: number;
+  totalFees: number;
+  medianFee: number;
+  feeRange: string;
+  reward: number;
+  poolId: number;
+  poolName: string;
+  poolSlug: string;
+  avgFee: number;
+  avgFeeRate: number;
+  coinbaseRaw: string;
+  coinbaseAddress: string;
+  coinbaseSignature: string;
+  coinbaseSignatureAscii: string;
+  avgTxSize: number;
+  totalInputs: number;
+  totalOutputs: number;
+  totalOutputAmt: number;
+  medianFeeAmt: number;
+  feePercentiles: string;
+  segwitTotalTxs: number;
+  segwitTotalSize: number;
+  segwitTotalWeight: number;
+  header: string;
+  utxoSetChange: number;
+  utxoSetSize: number;
+  totalInputAmt: number;
+}
+
 const BLOCK_DB_FIELDS = `
   blocks.hash AS id,
   blocks.height,
@@ -52,7 +94,7 @@ const BLOCK_DB_FIELDS = `
   blocks.header,
   blocks.utxoset_change AS utxoSetChange,
   blocks.utxoset_size AS utxoSetSize,
-  blocks.total_input_amt AS totalInputAmts
+  blocks.total_input_amt AS totalInputAmt
 `;
 
 class BlocksRepository {
@@ -171,6 +213,32 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Update missing fee amounts fields
+   *
+   * @param blockHash 
+   * @param feeAmtPercentiles 
+   * @param medianFeeAmt 
+   */
+  public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        JSON.stringify(feeAmtPercentiles),
+        medianFeeAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err(`Cannot update fee amounts for block ${blockHash}. Reason: ' + ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -432,7 +500,7 @@ class BlocksRepository {
 
       const blocks: BlockExtended[] = [];
       for (const block of rows) {
-        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
+        blocks.push(await this.formatDbBlockIntoExtendedBlock(block as DatabaseBlock));
       }
 
       return blocks;
@@ -459,37 +527,13 @@ class BlocksRepository {
         return null;
       }
 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
+      return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);  
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
-  /**
-   * Get one block by hash
-   */
-  public async $getBlockByHash(hash: string): Promise<object | null> {
-    try {
-      const query = `
-        SELECT ${BLOCK_DB_FIELDS}
-        FROM blocks
-        JOIN pools ON blocks.pool_id = pools.id
-        WHERE hash = ?;
-      `;
-      const [rows]: any[] = await DB.query(query, [hash]);
-
-      if (rows.length <= 0) {
-        return null;
-      }
- 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
-    } catch (e) {
-      logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   /**
    * Return blocks difficulty
    */
@@ -599,7 +643,6 @@ class BlocksRepository {
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
           logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
-          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
           await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
           return false;
@@ -619,7 +662,7 @@ class BlocksRepository {
    * Delete blocks from the database from blockHeight
    */
   public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks from height ${blockHeight} from the database`);
+    logger.info(`Delete newer blocks from height ${blockHeight} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
@@ -908,13 +951,32 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Save indexed effective fee statistics
+   * 
+   * @param id 
+   * @param feeStats 
+   */
+  public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET median_fee = ?, fee_span = ?
+        WHERE hash = ?`,
+        [feeStats.medianFee, JSON.stringify(feeStats.feeRange), id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Convert a mysql row block into a BlockExtended. Note that you
    * must provide the correct field into dbBlk object param
    * 
    * @param dbBlk 
    */
-  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
+  private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise<BlockExtended> {
     const blk: Partial<BlockExtended> = {};
     const extras: Partial<BlockExtension> = {};
 
@@ -978,6 +1040,7 @@ class BlocksRepository {
     }
 
     // If we're missing block summary related field, check if we can populate them on the fly now
+    // This is for example triggered upon re-org
     if (Common.blocksSummariesIndexingEnabled() &&
       (extras.medianFeeAmt === null || extras.feePercentiles === null))
     {
@@ -985,11 +1048,12 @@ class BlocksRepository {
       if (extras.feePercentiles === null) {
         const block = await bitcoinClient.getBlock(dbBlk.id, 2);
         const summary = blocks.summarizeBlock(block);
-        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+        await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
         extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
       }
       if (extras.feePercentiles !== null) {
         extras.medianFeeAmt = extras.feePercentiles[3];
+        await this.$updateFeeAmounts(dbBlk.id, extras.feePercentiles, extras.medianFeeAmt);
       }
     }
 

backend/src/repositories/BlocksSummariesRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { BlockSummary } from '../mempool.interfaces';
+import { BlockSummary, TransactionStripped } from '../mempool.interfaces';
 
 class BlocksSummariesRepository {
   public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
@@ -17,23 +17,17 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
-    const blockId = params.mined?.id;
+  public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
     try {
-      const transactions = JSON.stringify(params.mined?.transactions || []);
+      const transactionsStr = JSON.stringify(transactions);
       await DB.query(`
-        INSERT INTO blocks_summaries (height, id, transactions, template)
-        VALUE (?, ?, ?, ?)
-        ON DUPLICATE KEY UPDATE
-          transactions = ?
-      `, [params.height, blockId, transactions, '[]', transactions]);
+        INSERT INTO blocks_summaries
+        SET height = ?, transactions = ?, id = ?
+        ON DUPLICATE KEY UPDATE transactions = ?`,
+        [blockHeight, transactionsStr, blockId, transactionsStr]);
     } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
-      } else {
-        logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
-        throw e;
-      }
+      logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
     }
   }
 
@@ -68,19 +62,6 @@ class BlocksSummariesRepository {
     return [];
   }
 
-  /**
-   * Delete blocks from the database from blockHeight
-   */
-  public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
-
-    try {
-      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
-    } catch (e) {
-      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
   /**
    * Get the fee percentiles if the block has already been indexed, [] otherwise
    * 

backend/src/repositories/CpfpRepository.ts

@@ -48,7 +48,7 @@ class CpfpRepository {
     }
   }
 
-  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
+  public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise<boolean> {
     try {
       const clusterValues: any[] = [];
       const txs: any[] = [];

backend/src/repositories/HashratesRepository.ts

@@ -220,7 +220,7 @@ class HashratesRepository {
    * Delete hashrates from the database from timestamp
    */
   public async $deleteHashratesFromTimestamp(timestamp: number) {
-    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`);
+    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);

backend/src/repositories/PricesRepository.ts

@@ -160,7 +160,7 @@ class PricesRepository {
 
       // Compute fiat exchange rates
       let latestPrice = rates[0] as ApiPrice;
-      if (latestPrice.USD === -1) {
+      if (!latestPrice || latestPrice.USD === -1) {
         latestPrice = priceUpdater.getEmptyPricesObj();
       }
 

backend/src/tasks/lightning/forensics.service.ts

@@ -27,7 +27,7 @@ class ForensicsService {
 
   private async $runTasks(): Promise<void> {
     try {
-      logger.info(`Running forensics scans`);
+      logger.debug(`Running forensics scans`);
 
       if (config.MEMPOOL.BACKEND === 'esplora') {
         await this.$runClosedChannelsForensics(false);
@@ -73,7 +73,7 @@ class ForensicsService {
     let progress = 0;
 
     try {
-      logger.info(`Started running closed channel forensics...`);
+      logger.debug(`Started running closed channel forensics...`);
       let channels;
       if (onlyNewChannels) {
         channels = await channelsApi.$getClosedChannelsWithoutReason();
@@ -156,7 +156,7 @@ class ForensicsService {
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
-      logger.info(`Closed channels forensics scan complete.`);
+      logger.debug(`Closed channels forensics scan complete.`);
     } catch (e) {
       logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
     }
@@ -217,7 +217,7 @@ class ForensicsService {
     let progress = 0;
 
     try {
-      logger.info(`Started running open channel forensics...`);
+      logger.debug(`Started running open channel forensics...`);
       const channels = await channelsApi.$getChannelsWithoutSourceChecked();
 
       for (const openChannel of channels) {
@@ -266,7 +266,7 @@ class ForensicsService {
         }
       }
 
-      logger.info(`Open channels forensics scan complete.`);
+      logger.debug(`Open channels forensics scan complete.`);
     } catch (e) {
       logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
     } finally {

backend/src/tasks/lightning/network-sync.service.ts

@@ -283,7 +283,7 @@ class NetworkSyncService {
       } else {
         log += ` for the first time`;
       }
-      logger.info(`${log}`, logger.tags.ln);
+      logger.debug(`${log}`, logger.tags.ln);
 
       const channels = await channelsApi.$getChannelsByStatus([0, 1]);
       for (const channel of channels) {

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -15,16 +15,20 @@ class LightningStatsImporter {
   topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
 
   async $run(): Promise<void> {
-    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
-    logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
-    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+    try {
+      const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+      logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
+      await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
 
-    if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
-      return;
+      if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
+        return;
+      }
+
+      await this.$importHistoricalLightningStats();
+      await this.$cleanupIncorrectSnapshot();
+    } catch (e) {
+      logger.err(`Exception in LightningStatsImporter::$run(). ${e}`);
     }
-
-    await this.$importHistoricalLightningStats();
-    await this.$cleanupIncorrectSnapshot();
   }
 
   /**

backend/src/tasks/pools-updater.ts

@@ -62,7 +62,7 @@ class PoolsUpdater {
       if (this.currentSha === null) {
         logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {

backend/src/tasks/price-updater.ts

@@ -222,7 +222,7 @@ class PriceUpdater {
   private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
-    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
+    logger.debug(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
 
     const historicalPrices: PriceHistory[] = [];
 

backend/tsconfig.json

@@ -5,6 +5,7 @@
     "types": ["node", "jest"],
     "lib": ["es2019", "dom"],
     "strict": true,
+    "skipLibCheck": true,
     "noImplicitAny": false,
     "sourceMap": false,
     "outDir": "dist",

docker/README.md

@@ -204,7 +204,9 @@ Corresponding `docker-compose.yml` overrides:
 `mempool-config.json`:
 ```json
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
+    "REST_API_URL": "http://127.0.0.1:3000",
+    "UNIX_SOCKET_PATH": "/tmp/esplora-socket",
+    "RETRY_UNIX_SOCKET_AFTER": 30000
   },
 ```
 
@@ -213,6 +215,8 @@ Corresponding `docker-compose.yml` overrides:
   api:
     environment:
       ESPLORA_REST_API_URL: ""
+      ESPLORA_UNIX_SOCKET_PATH: ""
+      ESPLORA_RETRY_UNIX_SOCKET_AFTER: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -42,7 +42,9 @@
     "TLS_ENABLED": __ELECTRUM_TLS_ENABLED__
   },
   "ESPLORA": {
-    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+    "REST_API_URL": "__ESPLORA_REST_API_URL__",
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
+    "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",

docker/backend/start.sh

@@ -46,6 +46,8 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
 
 # ESPLORA
 __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
+__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null}
+__ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000}
 
 # SECOND_CORE_RPC
 __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
@@ -166,6 +168,8 @@ sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json
 sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config.json
 
 sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json
+sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json
+sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json
 
 sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -127,7 +127,7 @@ describe('Mainnet', () => {
 
         cy.get('.search-box-container > .form-control').type('S').then(() => {
           cy.wait('@search-1wizS');
-          cy.get('app-search-results button.dropdown-item').should('have.length', 5);
+          cy.get('app-search-results button.dropdown-item').should('have.length', 6);
         });
 
         cy.get('.search-box-container > .form-control').type('A').then(() => {

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mempool-frontend",
-  "version": "2.5.0",
+  "version": "2.6.0-dev",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "mempool-frontend",
-      "version": "2.5.0",
+      "version": "2.6.0-dev",
       "license": "GNU Affero General Public License v3.0",
       "dependencies": {
         "@angular-devkit/build-angular": "^14.2.10",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-frontend",
-  "version": "2.5.0",
+  "version": "2.6.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",

frontend/src/app/components/about/about.component.html

@@ -204,9 +204,9 @@
         <img class="image" src="/resources/profile/raspiblitz.svg" />
         <span>RaspiBlitz</span>
       </a>
-      <a href="https://github.com/mynodebtc/mynode" target="_blank" title="MyNode">
-        <img class="image" src="/resources/profile/mynodebtc.jpg" />
-        <span>MyNode</span>
+      <a href="https://github.com/mynodebtc/mynode" target="_blank" title="myNode">
+        <img class="image" src="/resources/profile/mynodebtc.png" />
+        <span>myNode</span>
       </a>
       <a href="https://github.com/RoninDojo/RoninDojo" target="_blank" title="RoninDojo">
         <img class="image" src="/resources/profile/ronindojo.png" />
@@ -253,7 +253,7 @@
         <span>Sparrow</span>
       </a>
       <a href="https://github.com/ACINQ/phoenix" target="_blank" title="Phoenix Wallet by ACINQ">
-        <img class="image" src="/resources/profile/phoenix.jpg" />
+        <img class="image not-rounded" src="/resources/profile/phoenix.svg" />
         <span>Phoenix</span>
       </a>
       <a href="https://github.com/lnbits/lnbits-legend" target="_blank" title="LNbits">

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -23,7 +23,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   @Input() unavailable: boolean = false;
   @Input() auditHighlighting: boolean = false;
   @Input() blockConversion: Price;
-  @Output() txClickEvent = new EventEmitter<TransactionStripped>();
+  @Output() txClickEvent = new EventEmitter<{ tx: TransactionStripped, keyModifier: boolean}>();
   @Output() txHoverEvent = new EventEmitter<string>();
   @Output() readyEvent = new EventEmitter();
 
@@ -326,7 +326,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     if (event.target === this.canvas.nativeElement && event.pointerType === 'touch') {
       this.setPreviewTx(event.offsetX, event.offsetY, true);
     } else if (event.target === this.canvas.nativeElement) {
-      this.onTxClick(event.offsetX, event.offsetY);
+      const keyMod = event.shiftKey || event.ctrlKey || event.metaKey;
+      const middleClick = event.which === 2 || event.button === 1;
+      this.onTxClick(event.offsetX, event.offsetY, keyMod || middleClick);
     }
   }
 
@@ -409,12 +411,12 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     }
   }
 
-  onTxClick(cssX: number, cssY: number) {
+  onTxClick(cssX: number, cssY: number, keyModifier: boolean = false) {
     const x = cssX * window.devicePixelRatio;
     const y = cssY * window.devicePixelRatio;
     const selected = this.scene.getTxAt({ x, y });
     if (selected && selected.txid) {
-      this.txClickEvent.emit(selected);
+      this.txClickEvent.emit({ tx: selected, keyModifier });
     }
   }
 

frontend/src/app/components/block/block.component.ts

@@ -612,9 +612,13 @@ export class BlockComponent implements OnInit, OnDestroy {
     });
   }
 
-  onTxClick(event: TransactionStripped): void {
-    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
-    this.router.navigate([url]);
+  onTxClick(event: { tx: TransactionStripped, keyModifier: boolean }): void {
+    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.tx.txid}`);
+    if (!event.keyModifier) {
+      this.router.navigate([url]);
+    } else {
+      window.open(url, '_blank');
+    }
   }
 
   onTxHover(txid: string): void {

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html

@@ -25,7 +25,7 @@
           </ng-template>
           <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-fee-span'" class="fee-span"
             *ngIf="block?.extras?.feeRange; else emptyfeespan">
-            {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{
+            {{ block?.extras?.feeRange?.[0] | number:feeRounding }} - {{
             block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} <ng-container
               i18n="shared.sat-vbyte|sat/vB">sat/vB</ng-container>
           </div>

frontend/src/app/components/liquid-master-page/liquid-master-page.component.html

@@ -90,6 +90,8 @@
 </nav>
 </header>
 
+<app-testnet-alert *ngIf="network.val === 'liquidtestnet'"></app-testnet-alert>
+
 <br />
 
 <router-outlet></router-outlet>

frontend/src/app/components/master-page/master-page.component.html

@@ -62,6 +62,8 @@
 </nav>
 </header>
 
+<app-testnet-alert *ngIf="network.val === 'testnet' || network.val === 'signet'"></app-testnet-alert>
+
 <br />
 
 <router-outlet></router-outlet>

frontend/src/app/components/master-page/master-page.component.scss

@@ -192,4 +192,4 @@ nav {
     margin: 33px 0px 0px -19px;
     font-size: 7px;
   }
-}
+}

frontend/src/app/components/master-page/master-page.component.ts

@@ -1,4 +1,4 @@
-import { Component, Inject, OnInit } from '@angular/core';
+import { Component, OnInit } from '@angular/core';
 import { Env, StateService } from '../../services/state.service';
 import { Observable, merge, of } from 'rxjs';
 import { LanguageService } from '../../services/language.service';

frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts

@@ -107,8 +107,12 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang
     this.isLoading$.next(false);
   }
 
-  onTxClick(event: TransactionStripped): void {
-    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
-    this.router.navigate([url]);
+  onTxClick(event: { tx: TransactionStripped, keyModifier: boolean }): void {
+    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.tx.txid}`);
+    if (!event.keyModifier) {
+      this.router.navigate([url]);
+    } else {
+      window.open(url, '_blank');
+    }
   }
 }

frontend/src/app/components/mempool-blocks/mempool-blocks.component.html

@@ -2,7 +2,7 @@
   <div class="mempool-blocks-container" [class.time-ltr]="timeLtr" *ngIf="(difficultyAdjustments$ | async) as da;">
     <div class="flashing">
       <ng-template ngFor let-projectedBlock [ngForOf]="mempoolBlocks$ | async" let-i="index" [ngForTrackBy]="trackByFn">
-        <div @blockEntryTrigger [@.disabled]="!animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
+        <div @blockEntryTrigger [@.disabled]="i > 0 || !animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
           <a draggable="false" [routerLink]="['/mempool-block/' | relativeUrl, i]"
             class="blockLink" [ngClass]="{'disabled': (this.stateService.blockScrolling$ | async)}">&nbsp;</a>
           <div class="block-body">

frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core';
+import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef, HostListener } from '@angular/core';
 import { Subscription, Observable, fromEvent, merge, of, combineLatest } from 'rxjs';
 import { MempoolBlock } from '../../interfaces/websocket.interface';
 import { StateService } from '../../services/state.service';
@@ -222,8 +222,13 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
     clearTimeout(this.resetTransitionTimeout);
   }
 
+  @HostListener('window:resize', ['$event'])
+  onResize(): void {
+    this.animateEntry = false;
+  }
+
   trackByFn(index: number, block: MempoolBlock) {
-    return (block.isStack) ? 'stack' : block.index;
+    return (block.isStack) ? `stack-${block.index}` : block.index;
   }
 
   reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {

frontend/src/app/components/start/start.component.ts

@@ -137,9 +137,11 @@ export class StartComponent implements OnInit, OnDestroy {
   }
 
   onMouseDown(event: MouseEvent) {
-    this.mouseDragStartX = event.clientX;
-    this.resetMomentum(event.clientX);
-    this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft;
+    if (!(event.which > 1 || event.button > 0)) {
+      this.mouseDragStartX = event.clientX;
+      this.resetMomentum(event.clientX);
+      this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft;
+    }
   }
   onPointerDown(event: PointerEvent) {
     if (this.isiOS) {

frontend/src/app/components/transaction/transaction-preview.component.html

@@ -8,10 +8,7 @@
     </div>
     <div *ngIf="network !== 'liquid' && network !== 'liquidtestnet'" class="features">
       <app-tx-features [tx]="tx"></app-tx-features>
-      <span *ngIf="cpfpInfo && (cpfpInfo.bestDescendant || cpfpInfo.descendants.length)" class="badge badge-primary mr-1">
-        CPFP
-      </span>
-      <span *ngIf="cpfpInfo && !cpfpInfo.bestDescendant && !cpfpInfo.descendants.length && cpfpInfo.ancestors.length" class="badge badge-info mr-1">
+      <span *ngIf="cpfpInfo && (cpfpInfo?.bestDescendant || cpfpInfo?.descendants?.length || cpfpInfo?.ancestors?.length)" class="badge badge-primary ml-1 mr-1">
         CPFP
       </span>
     </div>

frontend/src/app/components/transactions-list/transactions-list.component.html

@@ -1,3 +1,5 @@
+<div infiniteScroll [alwaysCallback]="true" [infiniteScrollDistance]="2" [infiniteScrollUpDistance]="1.5" [infiniteScrollThrottle]="50" (scrolled)="onScroll()">
+
 <ng-container *ngFor="let tx of transactions; let i = index; trackBy: trackByFn">
   <div *ngIf="!transactionPage" class="header-bg box tx-page-container">
     <a class="tx-link" [routerLink]="['/tx/' | relativeUrl, tx.txid]">
@@ -11,7 +13,7 @@
     </div>
   </div>
 
-  <div class="header-bg box" infiniteScroll [alwaysCallback]="true" [infiniteScrollDistance]="2" [infiniteScrollUpDistance]="1.5" [infiniteScrollThrottle]="50" (scrolled)="onScroll()" [attr.data-cy]="'tx-' + i">
+  <div class="header-bg box" [attr.data-cy]="'tx-' + i">
 
     <div *ngIf="errorUnblinded" class="error-unblinded">{{ errorUnblinded }}</div>
     <div class="row">
@@ -321,6 +323,8 @@
 
 </ng-container>
 
+</div>
+
 <ng-template #assetBox let-item>
   {{ item.value / pow(10, assetsMinimal[item.asset][3]) | number: '1.' + assetsMinimal[item.asset][3] + '-' + assetsMinimal[item.asset][3] }} {{ assetsMinimal[item.asset][1] }}
   <br />

frontend/src/app/components/transactions-list/transactions-list.component.ts

@@ -182,14 +182,7 @@ export class TransactionsListComponent implements OnInit, OnChanges {
   }
 
   onScroll(): void {
-    const scrollHeight = document.body.scrollHeight;
-    const scrollTop = document.documentElement.scrollTop;
-    if (scrollHeight > 0) {
-      const percentageScrolled = scrollTop * 100 / scrollHeight;
-      if (percentageScrolled > 50) {
-        this.loadMore.emit();
-      }
-    }
+    this.loadMore.emit();
   }
 
   haveBlindedOutputValues(tx: Transaction): boolean {

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html

@@ -29,7 +29,7 @@
   <ng-template #pegout>
     <ng-container *ngIf="line.pegout; else normal">
       <p *ngIf="!isConnector">Peg Out</p>
-      <p *ngIf="line.value != null"><app-amount [satoshis]="line.value"></app-amount></p>
+      <p *ngIf="line.displayValue != null"><app-amount [satoshis]="line.displayValue"></app-amount></p>
       <p class="address">
         <app-truncate [text]="line.pegout"></app-truncate>
       </p>
@@ -55,18 +55,18 @@
             <p *ngSwitchCase="'output'"><span i18n="transaction.input">Input</span>&nbsp; #{{ line.vin + 1 }}</p>
           </ng-container>
       </ng-container>
-      <p *ngIf="line.value == null && line.confidential" i18n="shared.confidential">Confidential</p>
-      <p *ngIf="line.value != null">
+      <p *ngIf="line.displayValue == null && line.confidential" i18n="shared.confidential">Confidential</p>
+      <p *ngIf="line.displayValue != null">
         <ng-template [ngIf]="line.asset && line.asset !== nativeAssetId" [ngIfElse]="defaultOutput">
           <div *ngIf="assetsMinimal && assetsMinimal[line.asset] else assetNotFound">
             <ng-container *ngTemplateOutlet="assetBox; context:{ $implicit: line }"></ng-container>
           </div>
           <ng-template #assetNotFound>
-            {{ line.value }} <span class="symbol">{{ line.asset | slice : 0 : 7 }}</span>
+            {{ line.displayValue }} <span class="symbol">{{ line.asset | slice : 0 : 7 }}</span>
           </ng-template>
         </ng-template>
         <ng-template #defaultOutput>
-          <app-amount [blockConversion]="blockConversion" [satoshis]="line.value"></app-amount>
+          <app-amount [blockConversion]="blockConversion" [satoshis]="line.displayValue"></app-amount>
         </ng-template>
       </p>
       <p *ngIf="line.type !== 'fee' && line.address" class="address">
@@ -76,5 +76,5 @@
 </div>
 
 <ng-template #assetBox let-item>
-  {{ item.value / pow(10, assetsMinimal[item.asset][3]) | number: '1.' + assetsMinimal[item.asset][3] + '-' + assetsMinimal[item.asset][3] }} <span class="symbol">{{ assetsMinimal[item.asset][1] }}</span>
+  {{ item.displayValue / pow(10, assetsMinimal[item.asset][3]) | number: '1.' + assetsMinimal[item.asset][3] + '-' + assetsMinimal[item.asset][3] }} <span class="symbol">{{ assetsMinimal[item.asset][1] }}</span>
 </ng-template>

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts

@@ -7,6 +7,7 @@ import { environment } from '../../../environments/environment';
 interface Xput {
   type: 'input' | 'output' | 'fee';
   value?: number;
+  displayValue?: number;
   index?: number;
   txid?: string;
   vin?: number;

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts

@@ -1,12 +1,13 @@
 import { Component, OnInit, Input, OnChanges, HostListener, Inject, LOCALE_ID } from '@angular/core';
 import { StateService } from '../../services/state.service';
-import { Outspend, Transaction } from '../../interfaces/electrs.interface';
+import { Outspend, Transaction, Vin, Vout } from '../../interfaces/electrs.interface';
 import { Router } from '@angular/router';
 import { ReplaySubject, merge, Subscription, of } from 'rxjs';
 import { tap, switchMap } from 'rxjs/operators';
 import { ApiService } from '../../services/api.service';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
 import { AssetsService } from '../../services/assets.service';
+import { environment } from '../../../environments/environment';
 
 interface SvgLine {
   path: string;
@@ -20,6 +21,7 @@ interface SvgLine {
 interface Xput {
   type: 'input' | 'output' | 'fee';
   value?: number;
+  displayValue?: number;
   index?: number;
   txid?: string;
   vin?: number;
@@ -74,6 +76,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
   zeroValueThickness = 20;
   hasLine: boolean;
   assetsMinimal: any;
+  nativeAssetId = this.stateService.network === 'liquidtestnet' ? environment.nativeTestAssetId : environment.nativeAssetId;
 
   outspendsSubscription: Subscription;
   refreshOutspends$: ReplaySubject<string> = new ReplaySubject();
@@ -167,7 +170,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     let voutWithFee = this.tx.vout.map((v, i) => {
       return {
         type: v.scriptpubkey_type === 'fee' ? 'fee' : 'output',
-        value: v?.value,
+        value: this.getOutputValue(v),
+        displayValue: v?.value,
         address: v?.scriptpubkey_address || v?.scriptpubkey_type?.toUpperCase(),
         index: i,
         pegout: v?.pegout?.scriptpubkey_address,
@@ -185,7 +189,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     let truncatedInputs = this.tx.vin.map((v, i) => {
       return {
         type: 'input',
-        value: v?.prevout?.value || (v?.is_coinbase && !totalValue ? 0 : undefined),
+        value: (v?.is_coinbase && !totalValue ? 0 : this.getInputValue(v)),
+        displayValue: v?.prevout?.value,
         txid: v.txid,
         vout: v.vout,
         address: v?.prevout?.scriptpubkey_address || v?.prevout?.scriptpubkey_type?.toUpperCase(),
@@ -229,14 +234,14 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
   }
 
   calcTotalValue(tx: Transaction): number {
-    const totalOutput = this.tx.vout.reduce((acc, v) => (v.value == null ? 0 : v.value) + acc, 0);
+    let totalOutput = this.tx.vout.reduce((acc, v) => (this.getOutputValue(v) || 0) + acc, 0);
     // simple sum of outputs + fee for bitcoin
     if (!this.isLiquid) {
       return this.tx.fee ? totalOutput + this.tx.fee : totalOutput;
     } else {
-      const totalInput = this.tx.vin.reduce((acc, v) => (v?.prevout?.value == null ? 0 : v.prevout.value) + acc, 0);
-      const confidentialInputCount = this.tx.vin.reduce((acc, v) => acc + (v?.prevout?.value == null ? 1 : 0), 0);
-      const confidentialOutputCount = this.tx.vout.reduce((acc, v) => acc + (v.value == null ? 1 : 0), 0);
+      const totalInput = this.tx.vin.reduce((acc, v) => (this.getInputValue(v) || 0) + acc, 0);
+      const confidentialInputCount = this.tx.vin.reduce((acc, v) => acc + (this.isUnknownInputValue(v) ? 1 : 0), 0);
+      const confidentialOutputCount = this.tx.vout.reduce((acc, v) => acc + (this.isUnknownOutputValue(v) ? 1 : 0), 0);
 
       // if there are unknowns on both sides, the total is indeterminate, so we'll just fudge it
       if (confidentialInputCount && confidentialOutputCount) {
@@ -456,6 +461,34 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     }
   }
 
+  getOutputValue(v: Vout): number | void {
+    if (!v) {
+      return null;
+    } else if (this.isLiquid && v.asset !== this.nativeAssetId) {
+      return null;
+    } else {
+      return v.value;
+    }
+  }
+
+  getInputValue(v: Vin): number | void {
+    if (!v?.prevout) {
+      return null;
+    } else if (this.isLiquid && v.prevout.asset !== this.nativeAssetId) {
+      return null;
+    } else {
+      return v.prevout.value;
+    }
+  }
+
+  isUnknownInputValue(v: Vin): boolean {
+    return v?.prevout?.value == null || this.isLiquid && v?.prevout?.asset !== this.nativeAssetId;
+  }
+
+  isUnknownOutputValue(v: Vout): boolean {
+    return v?.value == null || this.isLiquid && v?.asset !== this.nativeAssetId;
+  }
+
   @HostListener('pointermove', ['$event'])
   onPointerMove(event) {
     if (this.dir === 'rtl') {

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -8863,7 +8863,7 @@ export const faqData = [
     type: "endpoint",
     category: "advanced",
     showConditions: bitcoinNetworks,
-    fragment: "how-big-is-mempool-used-by-mempool.space",
+    fragment: "how-big-is-mempool-used-by-mempool-space",
     title: "How big is the mempool used by mempool.space?",
     options: { officialOnly: true },
   },

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -207,7 +207,7 @@
   <p>When a Bitcoin transaction is made, it is stored in a Bitcoin node's mempool before it is confirmed into a block. When the rate of incoming transactions exceeds the rate transactions are confirmed, the mempool grows in size.</p><p>By default, Bitcoin Core allocates 300MB of memory for its mempool, so when a node's mempool grows big enough to use all 300MB of allocated memory, we say it's "full".</p><p>Once a node's mempool is using all of its allocated memory, it will start rejecting new transactions below a certain feerate threshold—so when this is the case, be extra sure to set a feerate that (at a minimum) exceeds that threshold. The current threshold feerate (and memory usage) are displayed right on Mempool's front page.</p>
 </ng-template>
 
-<ng-template type="how-big-is-mempool-used-by-mempool.space">
+<ng-template type="how-big-is-mempool-used-by-mempool-space">
   <p>mempool.space uses multiple Bitcoin nodes to obtain data: some with the default 300MB mempool memory limit (call these Small Nodes) and others with a much larger mempool memory limit (call these Big Nodes).</p>
   <p>Many nodes on the Bitcoin network are configured to run with the default 300MB mempool memory setting. When all 300MB of memory are used up, such nodes will reject transactions below a certain threshold feerate. Running Small Nodes allows mempool.space to tell you what this threshold feerate is—this is the "Purging" feerate that shows on the front page when mempools are full, which you can use to be reasonably sure that your transaction will be widely propagated.</p>
     <p>Big Node mempools are so big that they don't need to reject (or purge) transactions. Such nodes allow for mempool.space to provide you with information on any pending transaction it has received—no matter how congested the mempool is, and no matter how low-feerate or low-priority the transaction is.</p>

frontend/src/app/fiat/fiat.component.html

@@ -1,4 +1,4 @@
-<span class="green-color" *ngIf="blockConversion; else noblockconversion">
+<span [class]="colorClass" *ngIf="blockConversion; else noblockconversion">
   {{
     (
       (blockConversion.price[currency] > -1 ? blockConversion.price[currency] : null) ??
@@ -8,7 +8,7 @@
 </span>
 
 <ng-template #noblockconversion>
-  <span class="green-color" *ngIf="(conversions$ | async) as conversions">
+  <span [class]="colorClass" *ngIf="(conversions$ | async) as conversions">
     {{ (conversions[currency] > -1 ? conversions[currency] : 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }}
   </span>
 </ng-template>

frontend/src/app/fiat/fiat.component.scss

@@ -1,3 +0,0 @@
-.green-color {
-  color: #3bcc49;
-}

frontend/src/app/fiat/fiat.component.ts

@@ -6,7 +6,7 @@ import { StateService } from '../services/state.service';
 @Component({
   selector: 'app-fiat',
   templateUrl: './fiat.component.html',
-  styleUrls: ['./fiat.component.scss'],
+  styleUrls: [],
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class FiatComponent implements OnInit, OnDestroy {
@@ -17,6 +17,7 @@ export class FiatComponent implements OnInit, OnDestroy {
   @Input() value: number;
   @Input() digitsInfo = '1.2-2';
   @Input() blockConversion: Price;
+  @Input() colorClass = 'green-color';
 
   constructor(
     private stateService: StateService,

frontend/src/app/lightning/node/node-preview.component.html

@@ -12,7 +12,7 @@
     </div>
   </div>
   <div class="row">
-    <div class="col-md">
+    <div class="col-md table-col">
       <a class="subtitle" [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">{{ node.public_key }}</a>
       <table class="table table-borderless table-striped">
         <tbody>

frontend/src/app/lightning/node/node-preview.component.scss

@@ -18,6 +18,10 @@
   }
 }
 
+.table-col {
+  max-width: calc(100% - 470px);
+}
+
 .map-col {
   flex-grow: 0;
   flex-shrink: 0;

frontend/src/app/services/storage.service.ts

@@ -12,20 +12,22 @@ export class StorageService {
 
   setDefaultValueIfNeeded(key: string, defaultValue: string) {
     const graphWindowPreference: string = this.getValue(key);
+    const fragment = window.location.hash.replace('#', '');
+
     if (graphWindowPreference === null) { // First visit to mempool.space
-      if (this.router.url.includes('graphs') && key === 'graphWindowPreference' ||
-        this.router.url.includes('pools') && key === 'miningWindowPreference'
+      if (window.location.pathname.includes('graphs') && key === 'graphWindowPreference' ||
+        window.location.pathname.includes('pools') && key === 'miningWindowPreference'
       ) {
-        this.setValue(key, this.route.snapshot.fragment ? this.route.snapshot.fragment : defaultValue);
+        this.setValue(key, fragment ? fragment : defaultValue);
       } else {
         this.setValue(key, defaultValue);
       }
-    } else if (this.router.url.includes('graphs') && key === 'graphWindowPreference' ||
-      this.router.url.includes('pools') && key === 'miningWindowPreference'
+    } else if (window.location.pathname.includes('graphs') && key === 'graphWindowPreference' ||
+      window.location.pathname.includes('pools') && key === 'miningWindowPreference'
     ) {
       // Visit a different graphs#fragment from last visit
-      if (this.route.snapshot.fragment !== null && graphWindowPreference !== this.route.snapshot.fragment) {
-        this.setValue(key, this.route.snapshot.fragment);
+      if (fragment !== null && graphWindowPreference !== fragment) {
+        this.setValue(key, fragment);
       }
     }
   }

frontend/src/app/shared/components/testnet-alert/testnet-alert.component.html

@@ -0,0 +1,8 @@
+<div class="container p-lg-0 pb-0" style="max-width: 100%; margin-top: 7px" *ngIf="storageService.getValue('hideWarning') !== 'hidden'">
+  <div class="alert alert-danger mb-0 text-center">
+    <div class="message-container" i18n="warning-testnet">This is a test network. Coins have no value.</div>
+    <button type="button" class="close" (click)="dismissWarning()">
+      <span aria-hidden="true">&times;</span>
+    </button>
+  </div>
+</div>

frontend/src/app/shared/components/testnet-alert/testnet-alert.component.scss

@@ -0,0 +1,31 @@
+.alert-danger {
+  color: #fff;
+  background-color: #b71c1c;
+  border-color: #b71c1c;
+  padding: 0.5rem 1.25rem;
+  margin: 0px 10px 0px 10px;
+
+  display: flex;
+  justify-content: center;
+}
+
+.message-container {
+  display: flex;
+  margin-left: auto;
+}
+
+.close {
+  display: flex;
+  color: #fff;
+  align-items: center;
+}
+
+button {
+  display: flex;
+  margin-left: auto;
+}
+
+span {
+  position: relative;
+  top: -2px;
+}

frontend/src/app/shared/components/testnet-alert/testnet-alert.component.ts

@@ -0,0 +1,20 @@
+import { ChangeDetectionStrategy, Component } from '@angular/core';
+import { StorageService } from '../../../services/storage.service';
+
+@Component({
+  selector: 'app-testnet-alert',
+  templateUrl: './testnet-alert.component.html',
+  styleUrls: ['./testnet-alert.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush,
+})
+export class TestnetAlertComponent {
+
+  constructor(
+    public storageService: StorageService,
+  ) { }
+
+  dismissWarning(): void {
+    this.storageService.setValue('hideWarning', 'hidden');
+  }
+
+}

frontend/src/app/shared/shared.module.ts

@@ -84,6 +84,7 @@ import { SearchResultsComponent } from '../components/search-form/search-results
 import { TimestampComponent } from './components/timestamp/timestamp.component';
 import { ToggleComponent } from './components/toggle/toggle.component';
 import { GeolocationComponent } from '../shared/components/geolocation/geolocation.component';
+import { TestnetAlertComponent } from './components/testnet-alert/testnet-alert.component';
 
 @NgModule({
   declarations: [
@@ -162,6 +163,7 @@ import { GeolocationComponent } from '../shared/components/geolocation/geolocati
     TimestampComponent,
     ToggleComponent,
     GeolocationComponent,
+    TestnetAlertComponent,
   ],
   imports: [
     CommonModule,

frontend/src/resources/profile/phoenix.svg

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   viewBox="0 0 62.449287 55.101883"
+   version="1.1"
+   id="svg395"
+   sodipodi:docname="phoenix.svg"
+   width="62.449287"
+   height="55.101883"
+   inkscape:version="1.2.2 (b0a8486541, 2022-12-01)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview397"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     showgrid="false"
+     inkscape:zoom="12.259259"
+     inkscape:cx="29.936556"
+     inkscape:cy="27.978852"
+     inkscape:window-width="3840"
+     inkscape:window-height="2091"
+     inkscape:window-x="0"
+     inkscape:window-y="32"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg395" />
+  <defs
+     id="defs389">
+    <style
+       id="style387">.outline{fill:#50b338;}.bg{fill:#fff;}</style>
+  </defs>
+  <path
+     class="outline"
+     d="m 58.070086,17.34 a 6,6 0 0 0 0.58,-0.65 7.16,7.16 0 0 0 1,-7 c -2.14,-6.51 -10.58,-9.69 -25.75,-9.69 -11.82,0 -23,0 -28.8400005,19.87 -7.53,25.62 -6.16,34.13 0.84,35.13 4.2400005,0.61 6.6200005,-1.62 9.0000005,-4 2,-2 4,-4.46 7,-4 14,2 24.1,-0.21 29.85,-4.83 a 7.7,7.7 0 0 0 3.15,-6.17 5.52,5.52 0 0 0 -0.49,-2.38 c 4.61,-1.11 7.58,-3.4 8,-7.76 0.33,-3.86 -1.45,-6.59 -4.34,-8.52 z m -0.67,8.07 c -0.43,4.74 -11.72,4.46 -26.44,3.59 a 1,1 0 0 0 -1.06,0.91 1,1 0 0 0 0.88,1.09 l 2.43,0.3 c 5.83,0.7 16.69,1.98 16.69,4.7 a 3.44,3.44 0 0 1 -1.41,2.56 c -4,3.4 -15.54,5.16 -24.77,3.76 -4.9,-0.74 -7.55,1.45 -9.9,3.38 -2.09,1.71 -3.73,3.06 -6.6800005,2.33 a 1.59,1.59 0 0 1 -1,-0.83 c -1.74,-3.2 -0.36,-12.85 3.76,-25.9 C 14.950086,5 22.490086,5 33.900086,5 c 12.22,0 19.69,2.24 21,6.32 a 2.17,2.17 0 0 1 -0.2,2.25 c -2.47,3.16 -15.44,1.86 -19.7,1.43 a 1.0054974,1.0054974 0 0 0 -0.21,2 l 2.06,0.23 c 8.4,0.88 21.1,2.22 20.55,8.18 z"
+     id="path391" />
+  <path
+     class="bg"
+     d="m 36.900086,17.22 -2.11,-0.22 a 1.0054974,1.0054974 0 0 1 0.21,-2 c 4.26,0.43 17.23,1.73 19.75,-1.43 a 2.17,2.17 0 0 0 0.2,-2.25 c -1.36,-4.08 -8.83,-6.32 -21.05,-6.32 -11.41,0 -19,0 -24.0000005,16.3 -4.12,13.05 -5.5,22.7 -3.72,25.9 a 1.59,1.59 0 0 0 1,0.83 c 2.9500005,0.73 4.5900005,-0.62 6.6800005,-2.33 2.35,-1.93 5,-4.12 9.9,-3.38 9.23,1.4 20.81,-0.36 24.77,-3.76 a 3.44,3.44 0 0 0 1.37,-2.56 c 0,-2.72 -10.86,-4 -16.69,-4.71 l -2.43,-0.29 a 1,1 0 0 1 -0.88,-1.09 1,1 0 0 1 1.06,-0.91 c 14.72,0.87 26,1.15 26.44,-3.59 0.55,-5.96 -12.15,-7.3 -20.5,-8.19 z"
+     id="path393" />
+</svg>

production/bitcoin.crontab

@@ -1,5 +1,5 @@
-@reboot sleep 30 ; screen -dmS mainnet /bitcoin/electrs/electrs-start-mainnet
-@reboot sleep 60 ; /usr/local/bin/bitcoind -testnet >/dev/null 2>&1
-@reboot sleep 70 ; screen -dmS testnet /bitcoin/electrs/electrs-start-testnet
-@reboot sleep 80 ; /usr/local/bin/bitcoind -signet >/dev/null 2>&1
-@reboot sleep 90 ; screen -dmS signet /bitcoin/electrs/electrs-start-signet
+@reboot screen -dmS mainnet /bitcoin/electrs/electrs-start-mainnet
+@reboot /usr/local/bin/bitcoind -testnet >/dev/null 2>&1
+@reboot screen -dmS testnet /bitcoin/electrs/electrs-start-testnet
+@reboot /usr/local/bin/bitcoind -signet >/dev/null 2>&1
+@reboot screen -dmS signet /bitcoin/electrs/electrs-start-signet

production/electrs-start-liquid

@@ -17,7 +17,7 @@ do
 		--db-dir __ELECTRS_DATA_ROOT__ \
 		--network liquid \
 		--daemon-dir "${HOME}" \
-		--http-addr '[::]:3001' \
+		--http-socket-file '/elements/socket/esplora-liquid-mainnet' \
 		--cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \
 		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt"
 	sleep 1

production/electrs-start-liquidtestnet

@@ -17,7 +17,7 @@ do
 		--db-dir __ELECTRS_DATA_ROOT__ \
 		--network liquidtestnet \
 		--daemon-dir "${HOME}" \
-		--http-addr '[::]:3004' \
+		--http-socket-file '/elements/socket/esplora-liquid-testnet' \
 		--cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \
 		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt"
 	sleep 1

production/electrs-start-mainnet

@@ -14,7 +14,7 @@ do
 		--cors '*' \
 		--db-dir __ELECTRS_DATA_ROOT__ \
 		--daemon-dir "${HOME}" \
-		--http-addr '[::]:3000' \
+		--http-socket-file '/bitcoin/socket/esplora-bitcoin-mainnet' \
 		--cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \
 		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt"
 

production/electrs-start-signet

@@ -16,7 +16,7 @@ do
 		--db-dir __ELECTRS_DATA_ROOT__ \
 		--daemon-rpc-addr '127.0.0.1:38332' \
 		--daemon-dir "${HOME}" \
-		--http-addr '[::]:3003' \
+		--http-socket-file '/bitcoin/socket/esplora-bitcoin-signet' \
 		--cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \
 		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt"
 	sleep 1

production/electrs-start-testnet

@@ -15,7 +15,7 @@ do
 		--cors '*' \
 		--db-dir __ELECTRS_DATA_ROOT__ \
 		--daemon-dir "${HOME}" \
-		--http-addr '[::]:3002' \
+		--http-socket-file '/bitcoin/socket/esplora-bitcoin-testnet' \
 		--cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \
 		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt"
 

production/elements.crontab

@@ -1,10 +1,10 @@
 # start elements on reboot
-@reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1
-@reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1
+@reboot /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1
+@reboot /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1
 
 # start electrs on reboot
-@reboot sleep 90 ; screen -dmS liquidv1 /elements/electrs/electrs-start-liquid
-@reboot sleep 90 ; screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet
+@reboot screen -dmS liquidv1 /elements/electrs/electrs-start-liquid
+@reboot screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet
 
 # hourly asset update and electrs restart
 6 * * * * cd $HOME/asset_registry_db && git pull --quiet origin master && cd $HOME/asset_registry_testnet_db && git pull --quiet origin master && killall electrs

production/install

@@ -192,6 +192,7 @@ case $OS in
         TOR_USER=_tor
         TOR_GROUP=_tor
         NGINX_USER=www
+        NGINX_GROUP=www
         NGINX_ETC_FOLDER=/usr/local/etc/nginx
         NGINX_CONFIGURATION=/usr/local/etc/nginx/nginx.conf
         CERTBOT_PKG=py39-certbot
@@ -209,6 +210,7 @@ case $OS in
         TOR_GROUP=debian-tor
         CERTBOT_PKG=python3-certbot-nginx
         NGINX_USER=www-data
+        NGINX_GROUP=www-data
         NGINX_ETC_FOLDER=/etc/nginx
         NGINX_CONFIGURATION=/etc/nginx/nginx.conf
     ;;
@@ -301,12 +303,6 @@ BISQ_HOME=/bisq
 # tor HS folder
 BISQ_TOR_HS=bisq
 
-# Unfurl user/group
-UNFURL_USER=unfurl
-UNFURL_GROUP=unfurl
-# Unfurl home folder
-UNFURL_HOME=/unfurl
-
 # liquid user/group
 ELEMENTS_USER=elements
 ELEMENTS_GROUP=elements
@@ -396,7 +392,7 @@ DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev liba
 # packages needed for mempool ecosystem
 FREEBSD_PKG=()
 FREEBSD_PKG+=(zsh sudo git git-lfs screen curl wget calc neovim)
-FREEBSD_PKG+=(openssh-portable py39-pip rust llvm90 jq base64 libzmq4)
+FREEBSD_PKG+=(openssh-portable py39-pip rust llvm10 jq base64 libzmq4)
 FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf)
 FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase)
 FREEBSD_PKG+=(geoipupdate)
@@ -547,6 +543,12 @@ zfsCreateFilesystems()
     zfs create -o "mountpoint=${ELEMENTS_HOME}/liquidv1" "${ZPOOL}/elements/liquidv1"
     zfs create -o "mountpoint=${ELEMENTS_ELECTRS_HOME}" "${ZPOOL}/elements/electrs"
 
+    # create /bitcoin/socket with custom ACL for electrs unix sockets
+    zfs create -o "mountpoint=${BITCOIN_HOME}/socket" "${ZPOOL}/bitcoin/socket"
+
+    # create /elements/socket with custom ACL for electrs unix sockets
+    zfs create -o "mountpoint=${ELEMENTS_HOME}/socket" "${ZPOOL}/elements/socket"
+
     # Bitcoin Mainnet
     if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
         for folder in chainstate indexes blocks
@@ -630,6 +632,7 @@ zfsCreateFilesystems()
 ext4CreateDir()
 {
     mkdir -p "/backup" "${ELEMENTS_HOME}" "${BITCOIN_HOME}" "${MINFEE_HOME}" "${ELECTRS_HOME}" "${MEMPOOL_HOME}" "${MYSQL_HOME}" "${BITCOIN_ELECTRS_HOME}" "${ELEMENTS_HOME}/liquidv1" "${ELEMENTS_ELECTRS_HOME}"
+
     # Bitcoin Mainnet
     if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
         for folder in chainstate indexes blocks
@@ -1019,7 +1022,7 @@ case $OS in
         osSudo "${ROOT_USER}" mkdir -p /usr/local/etc/syslog.d
         osSudo "${ROOT_USER}" install -c -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/mempool-logger" /usr/local/bin/mempool-logger
         osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/syslog.conf" /usr/local/etc/syslog.d/mempool.conf
-        
+
         echo "[*] Installing newsyslog configuration"
         osSudo "${ROOT_USER}" mkdir -p /usr/local/etc/newsyslog.conf.d
         osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/newsyslog-mempool-backend.conf" /usr/local/etc/newsyslog.conf.d/newsyslog-mempool-backend.conf
@@ -1057,17 +1060,8 @@ if [ "${TOR_INSTALL}" = ON ];then
     osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_CONFIGURATION}"
     osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}"
 
-    echo "[*] Adding Tor HS configuration for Mempool"
-    if [ "${MEMPOOL_ENABLE}" = "ON" ];then
-        if ! grep "${MEMPOOL_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
-            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${MEMPOOL_TOR_HS}/ >> ${TOR_CONFIGURATION}"
-            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:81 >> ${TOR_CONFIGURATION}"
-            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
-        fi
-    fi
-
     echo "[*] Adding Tor HS configuration for Bisq"
-    if [ "${BISQ_ENABLE}" = "ON" ];then
+    if [ "${BISQ_MAINNET_ENABLE}" = "ON" ];then
         if ! grep "${BISQ_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
             osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${BISQ_TOR_HS}/ >> ${TOR_CONFIGURATION}"
             osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:82 >> ${TOR_CONFIGURATION}"
@@ -1076,7 +1070,7 @@ if [ "${TOR_INSTALL}" = ON ];then
     fi
 
     echo "[*] Adding Tor HS configuration for Liquid"
-    if [ "${LIQUID_ENABLE}" = "ON" ];then
+    if [ "${ELEMENTS_LIQUID_ENABLE}" = "ON" ];then
         if ! grep "${LIQUID_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
             osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${LIQUID_TOR_HS}/ >> ${TOR_CONFIGURATION}"
             osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:83 >> ${TOR_CONFIGURATION}"
@@ -1273,25 +1267,25 @@ if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then
     if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
         osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}"
     fi
-    
+
     echo "[*] Cloning Liquid Electrs repo from ${ELEMENTS_ELECTRS_REPO_URL}"
     osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
     osSudo "${ELEMENTS_USER}" git clone --branch "${ELEMENTS_ELECTRS_REPO_BRANCH}" "${ELEMENTS_ELECTRS_REPO_URL}" "${ELEMENTS_HOME}/${ELEMENTS_ELECTRS_REPO_NAME}"
-    
+
     echo "[*] Checking out Liquid Electrs ${ELEMENTS_ELECTRS_LATEST_RELEASE}"
     osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_HOME}/${ELEMENTS_ELECTRS_REPO_NAME} && git checkout ${ELEMENTS_ELECTRS_LATEST_RELEASE}"
-    
+
     echo "[*] Cloning Liquid Asset Registry repo from ${LIQUID_ASSET_REGISTRY_DB_URL}"
     osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
     osSudo "${ELEMENTS_USER}" git clone "${LIQUID_ASSET_REGISTRY_DB_URL}" "${ELEMENTS_HOME}/${LIQUID_ASSET_REGISTRY_DB_NAME}"
-    
+
     echo "[*] Cloning Liquid Asset Registry testnet repo from ${LIQUIDTESTNET_ASSET_REGISTRY_DB_URL}"
     osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
     osSudo "${ELEMENTS_USER}" git clone "${LIQUIDTESTNET_ASSET_REGISTRY_DB_URL}" "${ELEMENTS_HOME}/${LIQUIDTESTNET_ASSET_REGISTRY_DB_NAME}"
-    
+
     echo "[*] Building Liquid Electrs release binary"
     osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
-    
+
     case $OS in
         FreeBSD)
             echo "[*] Patching Liquid Electrs code for FreeBSD"
@@ -1300,11 +1294,11 @@ if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then
         Debian)
         ;;
     esac
-    
+
     echo "[*] Building Liquid Electrs release binary"
     osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
 fi
-    
+
 ##############################
 # Core Lightning for Bitcoin #
 ##############################
@@ -1331,10 +1325,10 @@ case $OS in
         public_ipv4=$( ifconfig | grep 'inet ' | awk -F ' ' '{ print $2 }' | grep -v '^103\.165\.192\.' | grep -v '^127\.0\.0\.1' )
         public_ipv6=$( ifconfig | grep 'inet6' | awk -F ' ' '{ print $2 }' | grep -v '^2001:df6:7280::' | grep -v '^fe80::' | grep -v '^::1' )
 
-        crontab_cln+="@reboot sleep 60 ; screen -dmS main lightningd --rpc-file-mode 0660 --alias `hostname` --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
-        crontab_cln+="@reboot sleep 90 ; screen -dmS tes lightningd --rpc-file-mode 0660 --alias `hostname` --network testnet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
-        crontab_cln+="@reboot sleep 120 ; screen -dmS sig lightningd --rpc-file-mode 0660 --alias `hostname` --network signet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6 \n"
-        crontab_cln+="@reboot sleep 180 ; /mempool/mempool.space/lightning-seeder >/dev/null 2>&1\n"
+        crontab_cln+="@reboot sleep 10 ; screen -dmS main lightningd --rpc-file-mode 0660 --alias `hostname` --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
+        crontab_cln+="@reboot sleep 10 ; screen -dmS tes lightningd --rpc-file-mode 0660 --alias `hostname` --network testnet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
+        crontab_cln+="@reboot sleep 10 ; screen -dmS sig lightningd --rpc-file-mode 0660 --alias `hostname` --network signet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6 \n"
+        crontab_cln+="@reboot sleep 20 ; /mempool/mempool.space/lightning-seeder >/dev/null 2>&1\n"
         crontab_cln+="1 * * * * /mempool/mempool.space/lightning-seeder >/dev/null 2>&1\n"
         echo "${crontab_cln}" | crontab -u "${CLN_USER}" -
     ;;
@@ -1430,16 +1424,6 @@ fi
 
 if [ "${UNFURL_INSTALL}" = ON ];then
 
-    echo "[*] Creating Unfurl user"
-    osGroupCreate "${UNFURL_GROUP}"
-    osUserCreate "${UNFURL_USER}" "${UNFURL_HOME}" "${UNFURL_GROUP}"
-    osSudo "${ROOT_USER}" chsh -s `which zsh` "${UNFURL_USER}"
-
-    echo "[*] Creating Unfurl folder"
-    osSudo "${ROOT_USER}" mkdir -p "${UNFURL_HOME}"
-    osSudo "${ROOT_USER}" chown -R "${UNFURL_USER}:${UNFURL_GROUP}" "${UNFURL_HOME}"
-    osSudo "${UNFURL_USER}" touch "${UNFURL_HOME}/.zshrc"
-
     echo "[*] Insalling Unfurl source"
     case $OS in
 
@@ -1530,7 +1514,6 @@ if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
     case $OS in
 
         FreeBSD)
-            echo "[*] FIXME: Bitcoin Minfee service must be installed manually on FreeBSD"
         ;;
 
         Debian)
@@ -1548,7 +1531,6 @@ if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
     case $OS in
 
         FreeBSD)
-            echo "[*] FIXME: Bitcoin Testnet service must be installed manually on FreeBSD"
         ;;
 
         Debian)
@@ -1566,7 +1548,6 @@ if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
     case $OS in
 
         FreeBSD)
-            echo "[*] FIXME: Bitcoin Signet service must be installed manually on FreeBSD"
         ;;
 
         Debian)
@@ -1584,7 +1565,6 @@ if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
     case $OS in
 
         FreeBSD)
-            echo "[*] FIXME: Bitcoin Liquid service must be installed manually on FreeBSD"
         ;;
 
         Debian)
@@ -1602,7 +1582,6 @@ if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
     case $OS in
 
         FreeBSD)
-            echo "[*] FIXME: Bitcoin Liquid service must be installed manually on FreeBSD"
         ;;
 
         Debian)
@@ -1841,6 +1820,9 @@ case $OS in
     ;;
 esac
 
+# wait for mysql to start
+sleep 5
+
 mysql << _EOF_
 create database mempool;
 grant all on mempool.* to '${MEMPOOL_MAINNET_USER}'@'localhost' identified by '${MEMPOOL_MAINNET_PASS}';
@@ -1895,39 +1877,60 @@ chown "${MEMPOOL_USER}:${MEMPOOL_GROUP}" "${MEMPOOL_MYSQL_CREDENTIALS}"
 
 ##### nginx
 
+echo "[*] Adding Nginx configuration"
+osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
+mkdir -p /var/cache/nginx/services /var/cache/nginx/api
+chown "${NGINX_USER}:${NGINX_GROUP}" /var/cache/nginx/services /var/cache/nginx/api
+ln -s "${MEMPOOL_HOME}/mempool" "${NGINX_ETC_FOLDER}/mempool"
+osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}"
+osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"
+
+if [ "${TOR_INSTALL}" = ON ];then
+    echo "[*] Read tor v3 onion hostnames"
+
+    NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname")
+    osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+
+    if [ "${ELEMENTS_LIQUID_ENABLE}" = "ON" ];then
+        NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname")
+        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
+    fi
+
+    if [ "${BISQ_MAINNET_ENABLE}" = "ON" ];then
+        NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname")
+        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+    fi
+fi
+
+##### OS systemd
+
+echo "[*] Setting permissions for electrs sockets"
 case $OS in
 
     FreeBSD)
+        setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
+        chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
+        setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
+        chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"
     ;;
 
-Debian)
-        echo "[*] Adding Nginx configuration"
-        osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
-        mkdir -p /var/cache/nginx/services /var/cache/nginx/api
-        chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api
-        ln -s /mempool/mempool /etc/nginx/mempool
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}"
-        osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"
-        if [ "${TOR_INSTALL}" = ON ];then
-        echo "[*] Read tor v3 onion hostnames"
-            NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname")
-            NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname")
-            NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname")
-            osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-            osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
-            osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
-        fi
-        echo "[*] Restarting Nginx"
-        osSudo "${ROOT_USER}" service nginx restart
+    Debian)
     ;;
 esac
 
 ##### OS systemd
 
-echo "[*] Updating systemd daemon configuration"
+echo "[*] Updating system startup configuration"
 case $OS in
 
     FreeBSD)
+        echo 'nginx_enable="YES"' >> /etc/rc.conf
+        echo 'bitcoin_enable="YES"' >> /etc/rc.conf
+        echo 'tor_enable="YES"' >> /etc/rc.conf
+        echo 'postfix_enable="YES"' >> /etc/rc.conf
+        echo 'mysql_enable="YES"' >> /etc/rc.conf
+        echo 'mysql_dbdir="/mysql"' >> /etc/rc.conf
+        echo 'tor_enable="YES"' >> /etc/rc.conf
     ;;
 
     Debian)
@@ -1959,6 +1962,9 @@ case $OS in
     ;;
 esac
 
+echo "[*] Restarting Nginx"
+osSudo "${ROOT_USER}" service nginx restart
+
 ##### OS set Linux user ulimits
 
 echo "[*] Setting ulimits for users"
@@ -2060,20 +2066,12 @@ osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME} && ./upgrade" || true
 
 ##### finish
 
-case $OS in
-
-    FreeBSD)
-    ;;
-
-    Debian)
-        if [ "${TOR_INSTALL}" = ON ];then
-            echo "This are the generated Tor addresses:"
-            echo "${NGINX_MEMPOOL_ONION}"
-            echo "${NGINX_BISQ_ONION}"
-            echo "${NGINX_LIQUID_ONION}"
-        fi
-    ;;
-esac
+if [ "${TOR_INSTALL}" = ON ];then
+    echo "Your auto-generated Tor addresses are:"
+    echo "${NGINX_MEMPOOL_ONION}"
+    echo "${NGINX_BISQ_ONION}"
+    echo "${NGINX_LIQUID_ONION}"
+fi
 
 echo
 echo 'Please reboot to start all the services.'

production/mempool-config.bisq.json

@@ -15,12 +15,13 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4000"
+    "REST_API_URL": "http://127.0.0.1:5000",
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
   },
   "DATABASE": {
     "ENABLED": false,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_BISQ_USER__",
     "PASSWORD": "__MEMPOOL_BISQ_PASS__",
     "DATABASE": "mempool_bisq"

production/mempool-config.liquid.json

@@ -23,12 +23,13 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4001"
+    "REST_API_URL": "http://127.0.0.1:5001",
+    "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-mainnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_LIQUID_USER__",
     "PASSWORD": "__MEMPOOL_LIQUID_PASS__",
     "DATABASE": "mempool_liquid"

production/mempool-config.liquidtestnet.json

@@ -23,12 +23,13 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4004"
+    "REST_API_URL": "http://127.0.0.1:5004",
+    "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-testnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_LIQUIDTESTNET_USER__",
     "PASSWORD": "__MEMPOOL_LIQUIDTESTNET_PASS__",
     "DATABASE": "mempool_liquidtestnet"

production/mempool-config.mainnet-lightning.json

@@ -15,7 +15,8 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4000"
+    "REST_API_URL": "http://127.0.0.1:5000",
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
   },
   "LIGHTNING": {
     "ENABLED": true,
@@ -41,7 +42,7 @@
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_MAINNET_LIGHTNING_USER__",
     "PASSWORD": "__MEMPOOL_MAINNET_LIGHTNING_PASS__",
     "DATABASE": "mempool_mainnet_lightning"

production/mempool-config.mainnet.json

@@ -31,12 +31,13 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4000"
+    "REST_API_URL": "http://127.0.0.1:5000",
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_MAINNET_USER__",
     "PASSWORD": "__MEMPOOL_MAINNET_PASS__",
     "DATABASE": "mempool"

production/mempool-config.signet-lightning.json

@@ -15,7 +15,8 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4003"
+    "REST_API_URL": "http://127.0.0.1:5003",
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-signet"
   },
   "LIGHTNING": {
     "ENABLED": true,
@@ -36,7 +37,7 @@
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_SIGNET_LIGHTNING_USER__",
     "PASSWORD": "__MEMPOOL_SIGNET_LIGHTNING_PASS__",
     "DATABASE": "mempool_signet_lightning"

production/mempool-config.signet.json

@@ -22,12 +22,13 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4003"
+    "REST_API_URL": "http://127.0.0.1:5003",
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-signet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_SIGNET_USER__",
     "PASSWORD": "__MEMPOOL_SIGNET_PASS__",
     "DATABASE": "mempool_signet"

production/mempool-config.testnet-lightning.json

@@ -15,7 +15,8 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4002"
+    "REST_API_URL": "http://127.0.0.1:5002",
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-testnet"
   },
   "LIGHTNING": {
     "ENABLED": true,
@@ -36,7 +37,7 @@
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_TESTNET_LIGHTNING_USER__",
     "PASSWORD": "__MEMPOOL_TESTNET_LIGHTNING_PASS__",
     "DATABASE": "mempool_testnet_lightning"

production/mempool-config.testnet.json

@@ -22,12 +22,13 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4002"
+    "REST_API_URL": "http://127.0.0.1:5002",
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-testnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "USERNAME": "__MEMPOOL_TESTNET_USER__",
     "PASSWORD": "__MEMPOOL_TESTNET_PASS__",
     "DATABASE": "mempool_testnet"

production/mempool.crontab

@@ -1,5 +1,5 @@
 # start on reboot
-@reboot sleep 10 ; $HOME/start
+@reboot sleep 90 ; $HOME/start
 
 # daily backup
 37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 &

production/minfee.crontab

@@ -1 +1 @@
-@reboot sleep 120 ; /usr/local/bin/bitcoind >/dev/null 2>&1
+@reboot /usr/local/bin/bitcoind >/dev/null 2>&1

production/newsyslog-mempool-nginx.conf

@@ -1,8 +1,8 @@
-/var/log/nginx/access.log		nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
-/var/log/nginx/error.log		nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
-/var/log/nginx/bisq-access.log		nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
-/var/log/nginx/bisq-error.log		nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
-/var/log/nginx/liquid-access.log	nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
-/var/log/nginx/liquid-error.log		nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
-/var/log/nginx/mempool-access.log	nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
-/var/log/nginx/mempool-error.log	nobody:nobody	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/access.log		www:www	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/error.log		www:www	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/bisq-access.log		www:www	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/bisq-error.log		www:www	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/liquid-access.log	www:www	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/liquid-error.log		www:www	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/mempool-access.log	www:www	644  10   *    @T00  C     /var/run/mempool.pid 30
+/var/log/nginx/mempool-error.log	www:www	644  10   *    @T00  C     /var/run/mempool.pid 30

production/nginx/upstream-esplora.conf

@@ -1,15 +1,15 @@
 upstream esplora-bitcoin-mainnet {
-	server [::1]:3000 fail_timeout=10s max_fails=10 weight=99999;
+	server unix:/bitcoin/socket/esplora-bitcoin-mainnet fail_timeout=10s max_fails=10 weight=99999;
 }
 upstream esplora-liquid-mainnet {
-	server [::1]:3001 fail_timeout=10s max_fails=10 weight=99999;
+	server unix:/elements/socket/esplora-liquid-mainnet fail_timeout=10s max_fails=10 weight=99999;
 }
 upstream esplora-bitcoin-testnet {
-	server [::1]:3002 fail_timeout=10s max_fails=10 weight=99999;
+	server unix:/bitcoin/socket/esplora-bitcoin-testnet fail_timeout=10s max_fails=10 weight=99999;
 }
 upstream esplora-bitcoin-signet {
-	server [::1]:3003 fail_timeout=10s max_fails=10 weight=99999;
+	server unix:/bitcoin/socket/esplora-bitcoin-signet fail_timeout=10s max_fails=10 weight=99999;
 }
 upstream esplora-liquid-testnet {
-	server [::1]:3004 fail_timeout=10s max_fails=10 weight=99999;
+	server unix:/elements/socket/esplora-liquid-testnet fail_timeout=10s max_fails=10 weight=99999;
 }