attempt to recover from stalling the main loop

Backend block tip height endpoint

backend/mempool-config.sample.json

@@ -44,7 +44,8 @@
   },
   "ESPLORA": {
     "REST_API_URL": "http://127.0.0.1:3000",
-    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet"
+    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
+    "RETRY_UNIX_SOCKET_AFTER": 30000
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/__fixtures__/mempool-config.template.json

@@ -45,7 +45,8 @@
   },
   "ESPLORA": {
     "REST_API_URL": "__ESPLORA_REST_API_URL__",
-    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__"
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
+    "RETRY_UNIX_SOCKET_AFTER": "__ESPLORA_RETRY_UNIX_SOCKET_AFTER__"
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => {
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
 
-      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null });
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, RETRY_UNIX_SOCKET_AFTER: 30000 });
 
       expect(config.CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -94,6 +94,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
@@ -110,7 +111,6 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
-          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
@@ -589,10 +589,14 @@ class BitcoinRoutes {
     }
   }
 
-  private async getBlockTipHeight(req: Request, res: Response) {
+  private getBlockTipHeight(req: Request, res: Response) {
     try {
-      const result = await bitcoinApi.$getBlockHeightTip();
+      const result = blocks.getCurrentBlockHeight();
-      res.json(result);
+      if (!result) {
+        return res.status(503).send(`Service Temporarily Unavailable`);
+      }
+      res.setHeader('content-type', 'text/plain');
+      res.send(result.toString());
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/bitcoin/esplora-api.ts

@@ -3,102 +3,68 @@ import axios, { AxiosRequestConfig } from 'axios';
 import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
-import logger from '../../logger';
 
 const axiosConnection = axios.create({
   httpAgent: new http.Agent({ keepAlive: true, })
 });
 
 class ElectrsApi implements AbstractBitcoinApi {
-  private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
+  axiosConfig: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
     socketPath: config.ESPLORA.UNIX_SOCKET_PATH,
     timeout: 10000,
   } : {
     timeout: 10000,
   };
-  private axiosConfigTcpSocketOnly: AxiosRequestConfig = {
-    timeout: 10000,
-  };
 
-  unixSocketRetryTimeout;
+  constructor() { }
-  activeAxiosConfig;
-
-  constructor() {
-    this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
-  }
-
-  fallbackToTcpSocket() {
-    if (!this.unixSocketRetryTimeout) {
-      logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`);
-      // Retry the unix socket after a few seconds
-      this.unixSocketRetryTimeout = setTimeout(() => {
-        logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`);
-        this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
-        this.unixSocketRetryTimeout = undefined;
-      }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER);
-    }
-
-    // Use the TCP socket (reach a different esplora instance through nginx)
-    this.activeAxiosConfig = this.axiosConfigTcpSocketOnly;
-  }
-
-  $queryWrapper<T>(url, responseType = 'json'): Promise<T> {
-    return axiosConnection.get<T>(url, { ...this.activeAxiosConfig, responseType: responseType })
-      .then((response) => response.data)
-      .catch((e) => {
-        if (e?.code === 'ECONNREFUSED') {
-          this.fallbackToTcpSocket();
-          // Retry immediately
-          return axiosConnection.get<T>(url, this.activeAxiosConfig)
-            .then((response) => response.data)
-            .catch((e) => {
-              logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`);
-              throw e;
-            });
-        } else {
-          throw e;
-        }
-      });
-  }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return this.$queryWrapper<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids');
+    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return this.$queryWrapper<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId);
+    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex');
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return this.$queryWrapper<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height');
+    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getBlockHashTip(): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash');
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return this.$queryWrapper<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids');
+    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height);
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header');
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return this.$queryWrapper<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash);
+    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return this.$queryWrapper<any>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer')
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -119,11 +85,13 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return this.$queryWrapper<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout);
+    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+      .then((response) => response.data);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return this.$queryWrapper<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends');
+    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {

backend/src/api/blocks.ts

@@ -36,6 +36,8 @@ class Blocks {
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
   private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
+  private mainLoopTimeout: number = 120000;
+
   constructor() { }
 
   public getBlocks(): BlockExtended[] {
@@ -528,8 +530,12 @@ class Blocks {
   }
 
   public async $updateBlocks() {
+    // throw an error if this stalls the main loop for more than 2 minutes
+    const timer = this.startTimer();
+
     let fastForwarded = false;
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
+    this.updateTimerProgress(timer, 'got block height tip');
 
     if (this.blocks.length === 0) {
       this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1);
@@ -547,16 +553,21 @@ class Blocks {
 
     if (!this.lastDifficultyAdjustmentTime) {
       const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      this.updateTimerProgress(timer, 'got blockchain info for initial difficulty adjustment');
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
+        this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment');
         const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+        this.updateTimerProgress(timer, 'got block for initial difficulty adjustment');
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
+          this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
           const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
+          this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -571,9 +582,11 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
         await chainTips.updateOrphanedBlocks();
       }
 
+      this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
       const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
       const block = BitcoinApi.convertBlock(verboseBlock);
@@ -582,39 +595,51 @@ class Blocks {
       const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
+      this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
       // start async callbacks
+      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
       const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
 
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
+          this.updateTimerProgress(timer, `got block by height for ${this.currentBlockHeight}`);
           if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
             logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
             // We assume there won't be a reorg with more than 10 block depth
+            this.updateTimerProgress(timer, `rolling back diverged chain from ${this.currentBlockHeight}`);
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
+            this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
+              this.updateTimerProgress(timer, `reindexed block`);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
+              this.updateTimerProgress(timer, `reindexed block summary`);
               if (config.MEMPOOL.CPFP_INDEXING) {
                 await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+                this.updateTimerProgress(timer, `reindexed block cpfp`);
               }
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
+            this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
             logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
             indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
+          this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`);
 
           const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
           if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
             await blocksRepository.$saveBlockPrices([{
               height: blockExtended.height,
               priceId: lastestPriceId,
             }]);
+            this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
           } else {
             logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
             setTimeout(() => {
@@ -625,9 +650,11 @@ class Blocks {
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
+            this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
           }
           if (config.MEMPOOL.CPFP_INDEXING) {
             this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
+            this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
           }
         }
       }
@@ -640,6 +667,7 @@ class Blocks {
             difficulty: block.difficulty,
             adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
           });
+          this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
         }
 
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
@@ -664,7 +692,38 @@ class Blocks {
       }
 
       // wait for pending async callbacks to finish
+      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
       await Promise.all(callbackPromises);
+      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
+    }
+
+    this.clearTimer(timer);
+  }
+
+  private startTimer() {
+    const state: any = {
+      start: Date.now(),
+      progress: 'begin $updateBlocks',
+      timer: null,
+    };
+    state.timer = setTimeout(() => {
+      logger.err(`$updateBlocks stalled at "${state.progress}`);
+    }, this.mainLoopTimeout);
+    return state;
+  }
+
+  private updateTimerProgress(state, msg) {
+    state.progress = msg;
+    if (Date.now() - state.start > this.mainLoopTimeout) {
+      // abort the function if it already timed out
+      logger.err(`$updateBlocks attempted to resume after "${state.progress}"`);
+      throw new Error(`$updateBlocks attempted to resume after "${state.progress}"`);
+    }
+  }
+
+  private clearTimer(state) {
+    if (state.timer) {
+      clearTimeout(state.timer);
     }
   }
 

backend/src/api/mempool.ts

@@ -36,6 +36,8 @@ class Mempool {
   private timer = new Date().getTime();
   private missingTxCount = 0;
 
+  private mainLoopTimeout: number = 120000;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
   }
@@ -119,10 +121,15 @@ class Mempool {
 
   public async $updateMempool(): Promise<void> {
     logger.debug(`Updating mempool...`);
+
+    // throw an error if this stalls the main loop for more than 2 minutes
+    const timer = this.startTimer();
+
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
     const transactions = await bitcoinApi.$getRawMempool();
+    this.updateTimerProgress(timer, 'got raw mempool');
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
@@ -146,6 +153,7 @@ class Mempool {
       if (!this.mempoolCache[txid]) {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
+          this.updateTimerProgress(timer, 'fetched new transaction');
           this.mempoolCache[txid] = transaction;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
@@ -223,12 +231,43 @@ class Mempool {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
     if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      this.updateTimerProgress(timer, 'running async mempool callback');
       await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+      this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
     const end = new Date().getTime();
     const time = end - start;
     logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
+
+    this.clearTimer(timer);
+  }
+
+  private startTimer() {
+    const state: any = {
+      start: Date.now(),
+      progress: 'begin $updateMempool',
+      timer: null,
+    };
+    state.timer = setTimeout(() => {
+      logger.err(`$updateMempool stalled at "${state.progress}`);
+    }, this.mainLoopTimeout);
+    return state;
+  }
+
+  private updateTimerProgress(state, msg) {
+    state.progress = msg;
+    if (Date.now() - state.start > this.mainLoopTimeout) {
+      // abort the function if it already timed out
+      logger.err(`$updateMempool attempted to resume after "${state.progress}"`);
+      throw new Error(`$updateMempool attempted to resume after "${state.progress}"`);
+    }
+  }
+
+  private clearTimer(state) {
+    if (state.timer) {
+      clearTimeout(state.timer);
+    }
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {

backend/src/api/tx-selection-worker.ts

@@ -107,7 +107,7 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
 
     if (nextTx && !nextTx?.used) {
       // Check if the package fits into this block
-      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+      if (blocks.length >= 7 || (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS)) {
         const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
         // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
         const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
@@ -175,34 +175,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
       overflow = [];
     }
   }
-  // pack any leftover transactions into the last block
+
-  for (const tx of overflow) {
+  if (overflow.length > 0) {
-    if (!tx || tx?.used) {
+    logger.warn('GBT overflow list unexpectedly non-empty after final block constructed');
-      continue;
-    }
-    blockWeight += tx.weight;
-    const mempoolTx = mempool[tx.txid];
-    // update original copy of this tx with effective fee rate & relatives data
-    mempoolTx.effectiveFeePerVsize = tx.score;
-    if (tx.ancestorMap.size > 0) {
-      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
-      mempoolTx.cpfpRoot = tx.txid;
-    }
-    mempoolTx.cpfpChecked = true;
-    transactions.push(tx);
-    tx.used = true;
   }
-  const blockTransactions = transactions.map(t => mempool[t.txid]);
+  // add the final unbounded block if it contains any transactions
-  restOfArray.forEach(tx => {
+  if (transactions.length > 0) {
-    blockWeight += tx.weight;
+    blocks.push(transactions.map(t => mempool[t.txid]));
-    tx.effectiveFeePerVsize = tx.feePerVsize;
-    tx.cpfpChecked = false;
-    blockTransactions.push(tx);
-  });
-  if (blockTransactions.length) {
-    blocks.push(blockTransactions);
   }
-  transactions = [];
 
   const end = Date.now();
   const time = end - start;

backend/src/index.ts

@@ -45,8 +45,7 @@ class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
   private app: Application;
-  private currentBackendRetryInterval = 1;
+  private currentBackendRetryInterval = 5;
-  private backendRetryCount = 0;
 
   private maxHeapSize: number = 0;
   private heapLogInterval: number = 60;
@@ -180,22 +179,28 @@ class Server {
         }
       }
       memPool.deleteExpiredTransactions();
-      await blocks.$updateBlocks();
+      await Promise.race([
-      await memPool.$updateMempool();
+        blocks.$updateBlocks(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateBlocks timed out')), 180000))
+      ]);
+      await Promise.race([
+        memPool.$updateMempool(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateMempool timed out')), 180000))
+      ]);
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
-      this.backendRetryCount = 0;
+      this.currentBackendRetryInterval = 5;
     } catch (e: any) {
-      this.backendRetryCount++;
+      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
-      let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`;
       loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
       if (e?.stack) {
         loggerMsg += ` Stack trace: ${e.stack}`;
       }
       // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
       // From the second Exception, `logger.warn` the Exception and increase the retry delay
-      if (this.backendRetryCount >= 5) {
+      // Maximum retry delay is 60 seconds
+      if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
@@ -205,6 +210,8 @@ class Server {
         logger.debug(`AxiosError: ${e?.message}`);
       }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
+      this.currentBackendRetryInterval *= 2;
+      this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
     }
   }
 

docker/README.md

@@ -205,7 +205,8 @@ Corresponding `docker-compose.yml` overrides:
 ```json
   "ESPLORA": {
     "REST_API_URL": "http://127.0.0.1:3000",
-    "UNIX_SOCKET_PATH": "/tmp/esplora-socket"
+    "UNIX_SOCKET_PATH": "/tmp/esplora-socket",
+    "RETRY_UNIX_SOCKET_AFTER": 30000
   },
 ```
 
@@ -215,6 +216,7 @@ Corresponding `docker-compose.yml` overrides:
     environment:
       ESPLORA_REST_API_URL: ""
       ESPLORA_UNIX_SOCKET_PATH: ""
+      ESPLORA_RETRY_UNIX_SOCKET_AFTER: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -43,7 +43,8 @@
   },
   "ESPLORA": {
     "REST_API_URL": "__ESPLORA_REST_API_URL__",
-    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__"
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
+    "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",

docker/backend/start.sh

@@ -47,6 +47,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
 # ESPLORA
 __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
 __ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null}
+__ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000}
 
 # SECOND_CORE_RPC
 __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
@@ -168,6 +169,7 @@ sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config
 
 sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json
 sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json
+sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json
 
 sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts

@@ -22,7 +22,6 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
   @Input() offset: number = 0;
   @Input() height: number = 0; // max height of blocks in chunk (dynamic blocks only)
   @Input() count: number = 8; // number of blocks in this chunk (dynamic blocks only)
-  @Input() dynamicBlockCount: number = 8; // number of blocks in the dynamic block chunk
   @Input() loadingTip: boolean = false;
   @Input() connected: boolean = true;
   
@@ -46,6 +45,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
   feeRounding = '1.0-0';
   arrowVisible = false;
   arrowLeftPx = 30;
+  blocksFilled = false;
   arrowTransition = '1s';
   showMiningInfo = false;
   timeLtrSubscription: Subscription;
@@ -96,13 +96,14 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     this.tabHiddenSubscription = this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden);
     if (!this.static) {
       this.blocksSubscription = this.stateService.blocks$
-        .subscribe(([block, txConfirmed, batch]) => {
+        .subscribe(([block, txConfirmed]) => {
           if (this.blocks.some((b) => b.height === block.height)) {
             return;
           }
 
           if (this.blocks.length && block.height !== this.blocks[0].height + 1) {
             this.blocks = [];
+            this.blocksFilled = false;
           }
 
           this.blocks.unshift(block);
@@ -116,18 +117,20 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
           }
 
           this.blockStyles = [];
-          this.blocks.forEach((b, i) => {
+          if (this.blocksFilled && block.height > this.chainTip) {
-            if (i === 0 && !batch && block.height > this.chainTip) {
+            this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i, i ? -155 : -205)));
-              this.blockStyles.push(this.getStyleForBlock(b, i, -205));
+            setTimeout(() => {
-              setTimeout(() => {
+              this.blockStyles = [];
-                this.blockStyles = [];
+              this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i)));
-                this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i)));
+              this.cd.markForCheck();
-                this.cd.markForCheck();
+            }, 50);
-              }, 50);
+          } else {
-            } else {
+            this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i)));
-              this.blockStyles.push(this.getStyleForBlock(b, i));
+          }
-            }
+
-          });
+          if (this.blocks.length === this.dynamicBlocksAmount) {
+            this.blocksFilled = true;
+          }
 
           this.chainTip = Math.max(this.chainTip, block.height);
           this.cd.markForCheck();
@@ -157,7 +160,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
 
   ngOnChanges(changes: SimpleChanges): void {
     if (this.static) {
-      const animateSlide = (changes.dynamicBlockCount && changes.dynamicBlockCount.previousValue != null) || (changes.height && (changes.height.currentValue === changes.height.previousValue + 1));
+      const animateSlide = changes.height && (changes.height.currentValue === changes.height.previousValue + 1);
       this.updateStaticBlocks(animateSlide);
     }
   }
@@ -224,8 +227,8 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
         block = this.cacheService.getCachedBlock(height) || null;
       }
       this.blocks.push(block || {
-        placeholder: !isNaN(height) && height < 0,
+        placeholder: height < 0,
-        loading: isNaN(height) || height >= 0,
+        loading: height >= 0,
         id: '',
         height,
         version: 0,
@@ -308,7 +311,6 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     return {
       left: addLeft + (155 * index) + 'px',
       background: "#2d3348",
-      transition: animateEnterFrom ? 'background 2s, transform 1s' : null,
     };
   }
 
@@ -316,7 +318,6 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     const addLeft = animateEnterFrom || 0;
     return {
       left: addLeft + (155 * index) + 'px',
-      transition: animateEnterFrom ? 'background 2s, transform 1s' : null,
     };
   }
 
@@ -325,7 +326,6 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
 
     return {
       left: addLeft + 155 * this.emptyBlocks.indexOf(block) + 'px',
-      transition: animateEnterFrom ? 'background 2s, transform 1s' : null,
       background: "#2d3348",
     };
   }

frontend/src/app/components/blockchain/blockchain.component.html

@@ -6,7 +6,7 @@
         <app-mempool-blocks [hidden]="pageIndex > 0"></app-mempool-blocks>
         <app-blockchain-blocks [hidden]="pageIndex > 0"></app-blockchain-blocks>
         <ng-container *ngFor="let page of pages; trackBy: trackByPageFn">
-          <app-blockchain-blocks [static]="true" [offset]="page.offset" [height]="page.height" [dynamicBlockCount]="dynamicBlockCount" [count]="blocksPerPage" [loadingTip]="loadingTip" [connected]="connected"></app-blockchain-blocks>
+          <app-blockchain-blocks [static]="true" [offset]="page.offset" [height]="page.height" [count]="blocksPerPage" [loadingTip]="loadingTip" [connected]="connected"></app-blockchain-blocks>
         </ng-container>
       </div>
       <div id="divider" [hidden]="pageIndex > 0">

frontend/src/app/components/blockchain/blockchain.component.ts

@@ -12,7 +12,6 @@ export class BlockchainComponent implements OnInit, OnDestroy {
   @Input() pages: any[] = [];
   @Input() pageIndex: number;
   @Input() blocksPerPage: number = 8;
-  @Input() dynamicBlockCount: number = 8;
   @Input() minScrollWidth: number = 0;
 
   network: string;

frontend/src/app/components/mempool-blocks/mempool-blocks.component.html

@@ -2,7 +2,7 @@
   <div class="mempool-blocks-container" [class.time-ltr]="timeLtr" *ngIf="(difficultyAdjustments$ | async) as da;">
     <div class="flashing">
       <ng-template ngFor let-projectedBlock [ngForOf]="mempoolBlocks$ | async" let-i="index" [ngForTrackBy]="trackByFn">
-        <div @blockEntryTrigger [@.disabled]="!animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
+        <div @blockEntryTrigger [@.disabled]="i > 0 || !animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
           <a draggable="false" [routerLink]="['/mempool-block/' | relativeUrl, i]"
             class="blockLink" [ngClass]="{'disabled': (this.stateService.blockScrolling$ | async)}">&nbsp;</a>
           <div class="block-body">

frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core';
+import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef, HostListener } from '@angular/core';
 import { Subscription, Observable, fromEvent, merge, of, combineLatest } from 'rxjs';
 import { MempoolBlock } from '../../interfaces/websocket.interface';
 import { StateService } from '../../services/state.service';
@@ -105,6 +105,7 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
     });
     this.reduceMempoolBlocksToFitScreen(this.mempoolBlocks);
     this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden);
+    this.loadingBlocks$ = this.stateService.isLoadingWebSocket$;
 
     this.mempoolBlocks$ = merge(
       of(true),
@@ -140,13 +141,6 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
         })
       );
 
-    this.loadingBlocks$ = combineLatest([
-      this.stateService.isLoadingWebSocket$,
-      this.mempoolBlocks$
-    ]).pipe(map(([loading, mempoolBlocks]) => {
-      return loading || !mempoolBlocks.length;
-    }));
-
     this.difficultyAdjustments$ = this.stateService.difficultyAdjustment$
       .pipe(
         map((da) => {
@@ -228,8 +222,13 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
     clearTimeout(this.resetTransitionTimeout);
   }
 
+  @HostListener('window:resize', ['$event'])
+  onResize(): void {
+    this.animateEntry = false;
+  }
+
   trackByFn(index: number, block: MempoolBlock) {
-    return (block.isStack) ? 'stack' : block.index;
+    return (block.isStack) ? `stack-${block.index}` : block.index;
   }
 
   reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {

frontend/src/app/components/start/start.component.html

@@ -16,7 +16,7 @@
     (dragstart)="onDragStart($event)"
     (scroll)="onScroll($event)"
   >
-    <app-blockchain [pageIndex]="pageIndex" [pages]="pages" [dynamicBlockCount]="dynamicBlocksAmount" [blocksPerPage]="blocksPerPage" [minScrollWidth]="minScrollWidth"></app-blockchain>
+    <app-blockchain [pageIndex]="pageIndex" [pages]="pages" [blocksPerPage]="blocksPerPage" [minScrollWidth]="minScrollWidth"></app-blockchain>
   </div>
   <div class="reset-scroll" [class.hidden]="pageIndex === 0" (click)="resetScroll()">
     <fa-icon [icon]="['fas', 'circle-left']" [fixedWidth]="true"></fa-icon>

frontend/src/app/components/start/start.component.ts

@@ -2,7 +2,6 @@ import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild } fro
 import { Subscription } from 'rxjs';
 import { StateService } from '../../services/state.service';
 import { specialBlocks } from '../../app.constants';
-import { BlockExtended } from '../../interfaces/node-api.interface';
 
 @Component({
   selector: 'app-start',
@@ -30,8 +29,7 @@ export class StartComponent implements OnInit, OnDestroy {
   isMobile: boolean = false;
   isiOS: boolean = false;
   blockWidth = 155;
-  blocks: BlockExtended[] = [];
+  dynamicBlocksAmount: number = 8;
-  dynamicBlocksAmount: number;
   blockCount: number = 0;
   blocksPerPage: number = 1;
   pageWidth: number;
@@ -51,28 +49,15 @@ export class StartComponent implements OnInit, OnDestroy {
   }
 
   ngOnInit() {
-    this.firstPageWidth = 40 + (this.blockWidth * (this.dynamicBlocksAmount || 0));
+    this.firstPageWidth = 40 + (this.blockWidth * this.dynamicBlocksAmount);
-
+    this.blockCounterSubscription = this.stateService.blocks$.subscribe(() => {
-    this.blockCounterSubscription = this.stateService.blocks$.subscribe(([block, txConfirmed, batch]) => {
+      this.blockCount++;
-      if (this.blocks.some((b) => b.height === block.height)) {
+      this.dynamicBlocksAmount = Math.min(this.blockCount, this.stateService.env.KEEP_BLOCKS_AMOUNT, 8);
-        return;
-      }
-
-      if (this.blocks.length && block.height !== this.blocks[0].height + 1) {
-        this.blocks = [];
-      }
-
-      this.blocks.unshift(block);
-      this.blocks = this.blocks.slice(0, Math.min(8, this.stateService.env.KEEP_BLOCKS_AMOUNT));
-
-      this.dynamicBlocksAmount = this.blocks.length;
       this.firstPageWidth = 40 + (this.blockWidth * this.dynamicBlocksAmount);
-
+      if (this.blockCount <= Math.min(8, this.stateService.env.KEEP_BLOCKS_AMOUNT)) {
-      if (this.blocks.length <= Math.min(8, this.stateService.env.KEEP_BLOCKS_AMOUNT)) {
         this.onResize();
       }
     });
-
     this.onResize();
     this.updatePages();
     this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => {
@@ -152,9 +137,11 @@ export class StartComponent implements OnInit, OnDestroy {
   }
 
   onMouseDown(event: MouseEvent) {
-    this.mouseDragStartX = event.clientX;
+    if (!(event.which > 1 || event.button > 0)) {
-    this.resetMomentum(event.clientX);
+      this.mouseDragStartX = event.clientX;
-    this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft;
+      this.resetMomentum(event.clientX);
+      this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft;
+    }
   }
   onPointerDown(event: PointerEvent) {
     if (this.isiOS) {

frontend/src/app/services/state.service.ts

@@ -89,7 +89,7 @@ export class StateService {
 
   networkChanged$ = new ReplaySubject<string>(1);
   lightningChanged$ = new ReplaySubject<boolean>(1);
-  blocks$: ReplaySubject<[BlockExtended, boolean, boolean]>;
+  blocks$: ReplaySubject<[BlockExtended, boolean]>;
   transactions$ = new ReplaySubject<TransactionStripped>(6);
   conversions$ = new ReplaySubject<any>(1);
   bsqPrice$ = new ReplaySubject<number>(1);
@@ -157,7 +157,7 @@ export class StateService {
       }
     });
 
-    this.blocks$ = new ReplaySubject<[BlockExtended, boolean, boolean]>(this.env.KEEP_BLOCKS_AMOUNT);
+    this.blocks$ = new ReplaySubject<[BlockExtended, boolean]>(this.env.KEEP_BLOCKS_AMOUNT);
 
     if (this.env.BASE_MODULE === 'bisq') {
       this.network = this.env.BASE_MODULE;

frontend/src/app/services/websocket.service.ts

@@ -228,7 +228,7 @@ export class WebsocketService {
       blocks.forEach((block: BlockExtended) => {
         if (block.height > this.stateService.latestBlockHeight) {
           maxHeight = Math.max(maxHeight, block.height);
-          this.stateService.blocks$.next([block, false, true]);
+          this.stateService.blocks$.next([block, false]);
         }
       });
       this.stateService.updateChainTip(maxHeight);
@@ -241,7 +241,7 @@ export class WebsocketService {
     if (response.block) {
       if (response.block.height > this.stateService.latestBlockHeight) {
         this.stateService.updateChainTip(response.block.height);
-        this.stateService.blocks$.next([response.block, !!response.txConfirmed, false]);
+        this.stateService.blocks$.next([response.block, !!response.txConfirmed]);
       }
 
       if (response.txConfirmed) {