Don't delete transactions when checking if the current chain is valid

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -14,11 +14,11 @@ describe('Mempool Difficulty Adjustment', () => {
           750134,                         // Current block height
           0.6280047707459726,             // Previous retarget % (Passed through)
           'mainnet',                      // Network (if testnet, next value is non-zero)
-          0,                              // If not testnet, not used
+          0,                              // Latest block timestamp in seconds (only used if difficulty already locked in)
         ],
         { // Expected Result
           progressPercent: 9.027777777777777,
-          difficultyChange: 12.562233927411782,
+          difficultyChange: 13.180707740199772,
           estimatedRetargetDate: 1661895424692,
           remainingBlocks: 1834,
           remainingTime: 977591692,
@@ -41,7 +41,7 @@ describe('Mempool Difficulty Adjustment', () => {
         ],
         { // Expected Result is same other than timeOffset
           progressPercent: 9.027777777777777,
-          difficultyChange: 12.562233927411782,
+          difficultyChange: 13.180707740199772,
           estimatedRetargetDate: 1661895424692,
           remainingBlocks: 1834,
           remainingTime: 977591692,
@@ -54,6 +54,29 @@ describe('Mempool Difficulty Adjustment', () => {
           expectedBlocks: 161.68833333333333,
         },
       ],
+      [ // Vector 3 (mainnet lock-in (epoch ending 788255))
+        [ // Inputs
+          dt('2023-04-20T09:57:33.000Z'), // Last DA time (in seconds)
+          dt('2023-05-04T14:54:09.000Z'), // Current time (now) (in seconds)
+          788255,                         // Current block height
+          1.7220298879531821,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          dt('2023-05-04T14:54:26.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result
+          progressPercent: 99.95039682539682,
+          difficultyChange: -1.4512637555574193,
+          estimatedRetargetDate: 1683212658129,
+          remainingBlocks: 1,
+          remainingTime: 609129,
+          previousRetarget: 1.7220298879531821,
+          previousTime: 1681984653,
+          nextRetargetHeight: 788256,
+          timeAvg: 609129,
+          timeOffset: 0,
+          expectedBlocks: 2045.66,
+        },
+      ],
     ] as [[number, number, number, number, string, number], DifficultyAdjustment][];
 
     for (const vector of vectors) {

backend/src/api/audit.ts

@@ -93,17 +93,7 @@ class Audit {
       } else {
         if (!isDisplaced[tx.txid]) {
           added.push(tx.txid);
-        } else {
         }
-        let blockIndex = -1;
-        let index = -1;
-        projectedBlocks.forEach((block, bi) => {
-          const i = block.transactionIds.indexOf(tx.txid);
-          if (i >= 0) {
-            blockIndex = bi;
-            index = i;
-          }
-        });
         overflowWeight += tx.weight;
       }
       totalWeight += tx.weight;

backend/src/api/blocks.ts

@@ -520,6 +520,8 @@ class Blocks {
   }
 
   public async $updateBlocks() {
+    diskCache.lock();
+
     let fastForwarded = false;
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
 
@@ -581,11 +583,10 @@ class Blocks {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
           if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
-            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
+            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
             // We assume there won't be a reorg with more than 10 block depth
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
@@ -596,7 +597,7 @@ class Blocks {
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
-            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
             indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
@@ -658,6 +659,8 @@ class Blocks {
       // wait for pending async callbacks to finish
       await Promise.all(callbackPromises);
     }
+
+    diskCache.unlock();
   }
 
   /**
@@ -728,7 +731,7 @@ class Blocks {
 
     // Index the response if needed
     if (Common.blocksSummariesIndexingEnabled() === true) {
-      await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary});
+      await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions);
     }
 
     return summary.transactions;
@@ -844,7 +847,7 @@ class Blocks {
         if (cleanBlock.fee_amt_percentiles === null) {
           const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
           const summary = this.summarizeBlock(block);
-          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
           cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
         }
         if (cleanBlock.fee_amt_percentiles !== null) {

backend/src/api/difficulty-adjustment.ts

@@ -34,11 +34,12 @@ export function calcDifficultyAdjustment(
   const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
   const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
   const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
+  const actualTimespan = (blocksInEpoch === 2015 ? latestBlockTimestamp : nowSeconds) - DATime;
 
   let difficultyChange = 0;
   let timeAvgSecs = blocksInEpoch ? diffSeconds / blocksInEpoch : BLOCK_SECONDS_TARGET;
 
-  difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+  difficultyChange = (BLOCK_SECONDS_TARGET / (actualTimespan / (blocksInEpoch + 1)) - 1) * 100;
   // Max increase is x4 (+300%)
   if (difficultyChange > 300) {
     difficultyChange = 300;

backend/src/api/disk-cache.ts

@@ -18,6 +18,11 @@ class DiskCache {
   private static CHUNK_FILES = 25;
   private isWritingCache = false;
 
+  private semaphore: { resume: (() => void)[], locks: number } = {
+    resume: [],
+    locks: 0,
+  };
+
   constructor() {
     if (!cluster.isPrimary) {
       return;
@@ -73,6 +78,7 @@ class DiskCache {
           fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
         }
       } else {
+        await this.$yield();
         await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
           network: config.MEMPOOL.NETWORK,
           cacheSchemaVersion: this.cacheSchemaVersion,
@@ -82,6 +88,7 @@ class DiskCache {
           mempoolArray: mempoolArray.splice(0, chunkSize),
         }), { flag: 'w' });
         for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await this.$yield();
           await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
             mempool: {},
             mempoolArray: mempoolArray.splice(0, chunkSize),
@@ -124,7 +131,7 @@ class DiskCache {
     }
   }
 
-  loadMempoolCache(): void {
+  async $loadMempoolCache(): Promise<void> {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
@@ -168,13 +175,39 @@ class DiskCache {
         }
       }
 
-      memPool.setMempool(data.mempool);
+      await memPool.$setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
       blocks.setBlockSummaries(data.blockSummaries || []);
     } catch (e) {
       logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  private $yield(): Promise<void> {
+    if (this.semaphore.locks) {
+      logger.debug('Pause writing mempool and blocks data to disk cache (async)');
+      return new Promise((resolve) => {
+        this.semaphore.resume.push(resolve);
+      });
+    } else {
+      return Promise.resolve();
+    }
+  }
+
+  public lock(): void {
+    this.semaphore.locks++;
+  }
+
+  public unlock(): void {
+    this.semaphore.locks = Math.max(0, this.semaphore.locks - 1);
+    if (!this.semaphore.locks && this.semaphore.resume.length) {
+      const nextResume = this.semaphore.resume.shift();
+      if (nextResume) {
+        logger.debug('Resume writing mempool and blocks data to disk cache (async)');
+        nextResume();
+      }
+    }
+  }
 }
 
 export default new DiskCache();

backend/src/api/mempool-blocks.ts

@@ -1,5 +1,5 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, ThreadTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
 import { Worker } from 'worker_threads';
@@ -10,6 +10,9 @@ class MempoolBlocks {
   private mempoolBlockDeltas: MempoolBlockDelta[] = [];
   private txSelectionWorker: Worker | null = null;
 
+  private nextUid: number = 1;
+  private uidMap: Map<number, string> = new Map(); // map short numerical uids to full txids
+
   constructor() {}
 
   public getMempoolBlocks(): MempoolBlock[] {
@@ -87,19 +90,34 @@ class MempoolBlocks {
 
   private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    let blockWeight = 0;
     let blockSize = 0;
+    let blockWeight = 0;
+    let blockVsize = 0;
+    let blockFees = 0;
+    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
+    let transactionIds: string[] = [];
     let transactions: TransactionExtended[] = [];
     transactionsSorted.forEach((tx) => {
       if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
         || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
         blockWeight += tx.weight;
+        blockVsize += tx.vsize;
         blockSize += tx.size;
+        blockFees += tx.fee;
         transactions.push(tx);
+        transactionIds.push(tx.txid);
       } else {
         mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+        blockVsize = 0;
+        tx.position = {
+          block: mempoolBlocks.length,
+          vsize: blockVsize + (tx.vsize / 2),
+        };
+        blockVsize += tx.vsize;
         blockWeight = tx.weight;
         blockSize = tx.size;
+        blockFees = tx.fee;
+        transactionIds = [tx.txid];
         transactions = [tx];
       }
     });
@@ -147,19 +165,29 @@ class MempoolBlocks {
     return mempoolBlockDeltas;
   }
 
-  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+  public async $makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+    const start = Date.now();
+
+    // reset mempool short ids
+    this.resetUids();
+    for (const tx of Object.values(newMempool)) {
+      this.setUid(tx);
+    }
+
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
-    const strippedMempool: { [txid: string]: ThreadTransaction } = {};
-    Object.values(newMempool).filter(tx => !tx.deleteAfter).forEach(entry => {
-      strippedMempool[entry.txid] = {
-        txid: entry.txid,
-        fee: entry.fee,
-        weight: entry.weight,
-        feePerVsize: entry.fee / (entry.weight / 4),
-        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
-        vin: entry.vin.map(v => v.txid),
-      };
+    const strippedMempool: Map<number, CompactThreadTransaction> = new Map();
+    Object.values(newMempool).forEach(entry => {
+      if (entry.uid != null) {
+        strippedMempool.set(entry.uid, {
+          uid: entry.uid,
+          fee: entry.fee,
+          weight: entry.weight,
+          feePerVsize: entry.fee / (entry.weight / 4),
+          effectiveFeePerVsize: entry.effectiveFeePerVsize || (entry.fee / (entry.weight / 4)),
+          inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => uid != null) as number[],
+        });
+      }
     });
 
     // (re)initialize tx selection worker thread
@@ -178,7 +206,7 @@ class MempoolBlocks {
     // run the block construction algorithm in a separate thread, and wait for a result
     let threadErrorListener;
     try {
-      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+      const workerResultPromise = new Promise<{ blocks: number[][], rates: Map<number, number>, clusters: Map<number, number[]> }>((resolve, reject) => {
         threadErrorListener = reject;
         this.txSelectionWorker?.once('message', (result): void => {
           resolve(result);
@@ -186,123 +214,149 @@ class MempoolBlocks {
         this.txSelectionWorker?.once('error', reject);
       });
       this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
-      let { blocks, clusters } = await workerResultPromise;
-      // filter out stale transactions
-      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
-      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      if (filteredCount < unfilteredCount) {
-        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from makeBlockTemplates`);
-      }
+      const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      return this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults);
+      logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
+      return processed;
     } catch (e) {
       logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
     }
     return this.mempoolBlocks;
   }
 
-  public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
+  public async $updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: TransactionExtended[], saveResults: boolean = false): Promise<void> {
     if (!this.txSelectionWorker) {
       // need to reset the worker
-      this.makeBlockTemplates(newMempool, saveResults);
+      await this.$makeBlockTemplates(newMempool, saveResults);
       return;
     }
+
+    const start = Date.now();
+
+    for (const tx of Object.values(added)) {
+      this.setUid(tx);
+    }
+    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
-    const addedStripped: ThreadTransaction[] = added.map(entry => {
+    const addedStripped: CompactThreadTransaction[] = added.filter(entry => entry.uid != null).map(entry => {
       return {
-        txid: entry.txid,
+        uid: entry.uid || 0,
         fee: entry.fee,
         weight: entry.weight,
         feePerVsize: entry.fee / (entry.weight / 4),
-        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
-        vin: entry.vin.map(v => v.txid),
+        effectiveFeePerVsize: entry.effectiveFeePerVsize || (entry.fee / (entry.weight / 4)),
+        inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => uid != null) as number[],
       };
     });
 
     // run the block construction algorithm in a separate thread, and wait for a result
     let threadErrorListener;
     try {
-      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+      const workerResultPromise = new Promise<{ blocks: number[][], rates: Map<number, number>, clusters: Map<number, number[]> }>((resolve, reject) => {
         threadErrorListener = reject;
         this.txSelectionWorker?.once('message', (result): void => {
           resolve(result);
         });
         this.txSelectionWorker?.once('error', reject);
       });
-      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
-      let { blocks, clusters } = await workerResultPromise;
-      // filter out stale transactions
-      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
-      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      if (filteredCount < unfilteredCount) {
-        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from updateBlockTemplates`);
-      }
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedUids });
+      const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
+
+      this.removeUids(removedUids);
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
+      this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults);
+      logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
     } catch (e) {
       logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
     }
   }
 
-  private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] {
-    // update this thread's mempool with the results
-    blocks.forEach(block => {
-      block.forEach(tx => {
-        if (tx.txid && tx.txid in mempool) {
-          if (tx.effectiveFeePerVsize != null) {
-            mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
-          }
-          if (tx.cpfpRoot && tx.cpfpRoot in clusters) {
-            const ancestors: Ancestor[] = [];
-            const descendants: Ancestor[] = [];
-            const cluster = clusters[tx.cpfpRoot];
-            let matched = false;
-            cluster.forEach(txid => {
-              if (!txid || !mempool[txid]) {
-                logger.warn('projected transaction ancestor missing from mempool cache');
-                return;
-              }
-              if (txid === tx.txid) {
-                matched = true;
-              } else {
-                const relative = {
-                  txid: txid,
-                  fee: mempool[txid].fee,
-                  weight: mempool[txid].weight,
-                };
-                if (matched) {
-                  descendants.push(relative);
-                } else {
-                  ancestors.push(relative);
-                }
-              }
-            });
-            mempool[tx.txid].ancestors = ancestors;
-            mempool[tx.txid].descendants = descendants;
-            mempool[tx.txid].bestDescendant = null;
-          }
-          mempool[tx.txid].cpfpChecked = tx.cpfpChecked;
-        } else {
-          logger.warn('projected transaction missing from mempool cache');
-        }
-      });
-    });
+  private processBlockTemplates(mempool, blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }, saveResults): MempoolBlockWithTransactions[] {
+    for (const txid of Object.keys(rates)) {
+      if (txid in mempool) {
+        mempool[txid].effectiveFeePerVsize = rates[txid];
+      }
+    }
 
-    // unpack the condensed blocks into proper mempool blocks
-    const mempoolBlocks = blocks.map((transactions, blockIndex) => {
-      return this.dataToMempoolBlocks(transactions.map(tx => {
-        return mempool[tx.txid] || null;
-      }).filter(tx => !!tx), blockIndex);
-    });
+    const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees }[] = [];
+    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
+    // update this thread's mempool with the results
+    for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+      const block: string[] = blocks[blockIndex];
+      let txid: string;
+      let mempoolTx: TransactionExtended;
+      let totalSize = 0;
+      let totalVsize = 0;
+      let totalWeight = 0;
+      let totalFees = 0;
+      const transactions: TransactionExtended[] = [];
+      for (let txIndex = 0; txIndex < block.length; txIndex++) {
+        txid = block[txIndex];
+        if (txid) {
+          mempoolTx = mempool[txid];
+          // save position in projected blocks
+          mempoolTx.position = {
+            block: blockIndex,
+            vsize: totalVsize + (mempoolTx.vsize / 2),
+          };
+          mempoolTx.cpfpChecked = true;
+
+          totalSize += mempoolTx.size;
+          totalVsize += mempoolTx.vsize;
+          totalWeight += mempoolTx.weight;
+          totalFees += mempoolTx.fee;
+
+          transactions.push(mempoolTx);
+        }
+      }
+      readyBlocks.push({
+        transactionIds: block,
+        transactions,
+        totalSize,
+        totalWeight,
+        totalFees
+      });
+    }
+
+    for (const cluster of Object.values(clusters)) {
+      for (const memberTxid of cluster) {
+        if (memberTxid in mempool) {
+          const mempoolTx = mempool[memberTxid];
+          const ancestors: Ancestor[] = [];
+          const descendants: Ancestor[] = [];
+          let matched = false;
+          cluster.forEach(txid => {
+            if (txid === memberTxid) {
+              matched = true;
+            } else {
+              const relative = {
+                txid: txid,
+                fee: mempool[txid].fee,
+                weight: mempool[txid].weight,
+              };
+              if (matched) {
+                descendants.push(relative);
+              } else {
+                ancestors.push(relative);
+              }
+            }
+          });
+          mempoolTx.ancestors = ancestors;
+          mempoolTx.descendants = descendants;
+          mempoolTx.bestDescendant = null;
+        }
+      }
+    }
+
+    const mempoolBlocks = readyBlocks.map((b, i) => this.dataToMempoolBlocks(b.transactions, i));
 
     if (saveResults) {
       const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
@@ -344,6 +398,56 @@ class MempoolBlocks {
       transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
+
+  private resetUids(): void {
+    this.uidMap.clear();
+    this.nextUid = 1;
+  }
+
+  private setUid(tx: TransactionExtended): number {
+    const uid = this.nextUid;
+    this.nextUid++;
+    this.uidMap.set(uid, tx.txid);
+    tx.uid = uid;
+    return uid;
+  }
+
+  private getUid(tx: TransactionExtended): number | void {
+    if (tx?.uid != null && this.uidMap.has(tx.uid)) {
+      return tx.uid;
+    }
+  }
+
+  private removeUids(uids: number[]): void {
+    for (const uid of uids) {
+      this.uidMap.delete(uid);
+    }
+  }
+
+  private convertResultTxids({ blocks, rates, clusters }: { blocks: number[][], rates: Map<number, number>, clusters: Map<number, number[]>})
+    : { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }} {
+    const convertedBlocks: string[][] = blocks.map(block => block.map(uid => {
+      return this.uidMap.get(uid) || '';
+    }));
+    const convertedRates = {};
+    for (const rateUid of rates.keys()) {
+      const rateTxid = this.uidMap.get(rateUid);
+      if (rateTxid) {
+        convertedRates[rateTxid] = rates.get(rateUid);
+      }
+    }
+    const convertedClusters = {};
+    for (const rootUid of clusters.keys()) {
+      const rootTxid = this.uidMap.get(rootUid);
+      if (rootTxid) {
+        const members = clusters.get(rootUid)?.map(uid => {
+          return this.uidMap.get(uid);
+        });
+        convertedClusters[rootTxid] = members;
+      }
+    }
+    return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
+  }
 }
 
 export default new MempoolBlocks();

backend/src/api/mempool.ts

@@ -20,7 +20,7 @@ class Mempool {
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
-  private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+  private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => Promise<void>) | undefined;
 
   private txPerSecondArray: number[] = [];
@@ -71,20 +71,20 @@ class Mempool {
 
   public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
     newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise<void>) {
-    this.asyncMempoolChangedCallback = fn;
+    this.$asyncMempoolChangedCallback = fn;
   }
 
   public getMempool(): { [txid: string]: TransactionExtended } {
     return this.mempoolCache;
   }
 
-  public setMempool(mempoolData: { [txId: string]: TransactionExtended }) {
+  public async $setMempool(mempoolData: { [txId: string]: TransactionExtended }) {
     this.mempoolCache = mempoolData;
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
     }
-    if (this.asyncMempoolChangedCallback) {
-      this.asyncMempoolChangedCallback(this.mempoolCache, [], []);
+    if (this.$asyncMempoolChangedCallback) {
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, [], []);
     }
   }
 
@@ -222,8 +222,8 @@ class Mempool {
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
-    if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
 
     const end = new Date().getTime();

backend/src/api/tx-selection-worker.ts

@@ -1,11 +1,11 @@
 import config from '../config';
 import logger from '../logger';
-import { ThreadTransaction, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces';
+import { CompactThreadTransaction, AuditTransaction } from '../mempool.interfaces';
 import { PairingHeap } from '../utils/pairing-heap';
 import { Common } from './common';
 import { parentPort } from 'worker_threads';
 
-let mempool: { [txid: string]: ThreadTransaction } = {};
+let mempool: Map<number, CompactThreadTransaction> = new Map();
 
 if (parentPort) {
   parentPort.on('message', (params) => {
@@ -13,18 +13,18 @@ if (parentPort) {
       mempool = params.mempool;
     } else if (params.type === 'update') {
       params.added.forEach(tx => {
-        mempool[tx.txid] = tx;
+        mempool.set(tx.uid, tx);
       });
-      params.removed.forEach(txid => {
-        delete mempool[txid];
+      params.removed.forEach(uid => {
+        mempool.delete(uid);
       });
     }
     
-    const { blocks, clusters } = makeBlockTemplates(mempool);
+    const { blocks, rates, clusters } = makeBlockTemplates(mempool);
 
     // return the result to main thread.
     if (parentPort) {
-     parentPort.postMessage({ blocks, clusters });
+      parentPort.postMessage({ blocks, rates, clusters });
     }
   });
 }
@@ -33,26 +33,25 @@ if (parentPort) {
 * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
 * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
 */
-function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
-  : { blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } } {
+function makeBlockTemplates(mempool: Map<number, CompactThreadTransaction>)
+  : { blocks: number[][], rates: Map<number, number>, clusters: Map<number, number[]> } {
   const start = Date.now();
-  const auditPool: { [txid: string]: AuditTransaction } = {};
+  const auditPool: Map<number, AuditTransaction> = new Map();
   const mempoolArray: AuditTransaction[] = [];
-  const restOfArray: ThreadTransaction[] = [];
-  const cpfpClusters: { [root: string]: string[] } = {};
+  const cpfpClusters: Map<number, number[]> = new Map();
   
-  // grab the top feerate txs up to maxWeight
-  Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+  mempool.forEach(tx => {
+    tx.dirty = false;
     // initializing everything up front helps V8 optimize property access later
-    auditPool[tx.txid] = {
-      txid: tx.txid,
+    auditPool.set(tx.uid, {
+      uid: tx.uid,
       fee: tx.fee,
       weight: tx.weight,
       feePerVsize: tx.feePerVsize,
       effectiveFeePerVsize: tx.feePerVsize,
-      vin: tx.vin,
+      inputs: tx.inputs || [],
       relativesSet: false,
-      ancestorMap: new Map<string, AuditTransaction>(),
+      ancestorMap: new Map<number, AuditTransaction>(),
       children: new Set<AuditTransaction>(),
       ancestorFee: 0,
       ancestorWeight: 0,
@@ -60,8 +59,8 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
       used: false,
       modified: false,
       modifiedNode: null,
-    };
-    mempoolArray.push(auditPool[tx.txid]);
+    });
+    mempoolArray.push(auditPool.get(tx.uid) as AuditTransaction);
   });
 
   // Build relatives graph & calculate ancestor scores
@@ -72,15 +71,28 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
   }
 
   // Sort by descending ancestor score
-  mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+  mempoolArray.sort((a, b) => {
+    if (b.score === a.score) {
+      // tie-break by uid for stability
+      return a.uid < b.uid ? -1 : 1;
+    } else {
+      return (b.score || 0) - (a.score || 0);
+    }
+  });
 
   // Build blocks by greedily choosing the highest feerate package
   // (i.e. the package rooted in the transaction with the best ancestor score)
-  const blocks: ThreadTransaction[][] = [];
+  const blocks: number[][] = [];
   let blockWeight = 4000;
-  let blockSize = 0;
   let transactions: AuditTransaction[] = [];
-  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => {
+    if (a.score === b.score) {
+      // tie-break by uid for stability
+      return a.uid > b.uid;
+    } else {
+      return (a.score || 0) > (b.score || 0);
+    }
+  });
   let overflow: AuditTransaction[] = [];
   let failures = 0;
   let top = 0;
@@ -107,30 +119,36 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
 
     if (nextTx && !nextTx?.used) {
       // Check if the package fits into this block
-      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+      if (blocks.length >= 7 || (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS)) {
         const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
         // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
         const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
         let isCluster = false;
         if (sortedTxSet.length > 1) {
-          cpfpClusters[nextTx.txid] = sortedTxSet.map(tx => tx.txid);
+          cpfpClusters.set(nextTx.uid, sortedTxSet.map(tx => tx.uid));
           isCluster = true;
         }
         const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
         const used: AuditTransaction[] = [];
         while (sortedTxSet.length) {
           const ancestor = sortedTxSet.pop();
-          const mempoolTx = mempool[ancestor.txid];
+          const mempoolTx = mempool.get(ancestor.uid);
+          if (!mempoolTx) {
+            continue;
+          }
           ancestor.used = true;
-          ancestor.usedBy = nextTx.txid;
+          ancestor.usedBy = nextTx.uid;
           // update original copy of this tx with effective fee rate & relatives data
-          mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
-          if (isCluster) {
-            mempoolTx.cpfpRoot = nextTx.txid;
+          if (mempoolTx.effectiveFeePerVsize !== effectiveFeeRate) {
+            mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+            mempoolTx.dirty = true;
+          }
+          if (mempoolTx.cpfpRoot !== nextTx.uid) {
+            mempoolTx.cpfpRoot = isCluster ? nextTx.uid : null;
+            mempoolTx.dirty;
           }
           mempoolTx.cpfpChecked = true;
           transactions.push(ancestor);
-          blockSize += ancestor.size;
           blockWeight += ancestor.weight;
           used.push(ancestor);
         }
@@ -156,11 +174,10 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
     if ((exceededPackageTries || queueEmpty) && blocks.length < 7) {
       // construct this block
       if (transactions.length) {
-        blocks.push(transactions.map(t => mempool[t.txid]));
+        blocks.push(transactions.map(t => t.uid));
       }
       // reset for the next block
       transactions = [];
-      blockSize = 0;
       blockWeight = 4000;
 
       // 'overflow' packages didn't fit in this block, but are valid candidates for the next
@@ -175,50 +192,38 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
       overflow = [];
     }
   }
-  // pack any leftover transactions into the last block
-  for (const tx of overflow) {
-    if (!tx || tx?.used) {
-      continue;
-    }
-    blockWeight += tx.weight;
-    const mempoolTx = mempool[tx.txid];
-    // update original copy of this tx with effective fee rate & relatives data
-    mempoolTx.effectiveFeePerVsize = tx.score;
-    if (tx.ancestorMap.size > 0) {
-      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
-      mempoolTx.cpfpRoot = tx.txid;
-    }
-    mempoolTx.cpfpChecked = true;
-    transactions.push(tx);
-    tx.used = true;
+
+  if (overflow.length > 0) {
+    logger.warn('GBT overflow list unexpectedly non-empty after final block constructed');
   }
-  const blockTransactions = transactions.map(t => mempool[t.txid]);
-  restOfArray.forEach(tx => {
-    blockWeight += tx.weight;
-    tx.effectiveFeePerVsize = tx.feePerVsize;
-    tx.cpfpChecked = false;
-    blockTransactions.push(tx);
-  });
-  if (blockTransactions.length) {
-    blocks.push(blockTransactions);
+  // add the final unbounded block if it contains any transactions
+  if (transactions.length > 0) {
+    blocks.push(transactions.map(t => t.uid));
+  }
+
+  // get map of dirty transactions
+  const rates = new Map<number, number>();
+  for (const tx of mempool.values()) {
+    if (tx?.dirty) {
+      rates.set(tx.uid, tx.effectiveFeePerVsize || tx.feePerVsize);
+    }
   }
-  transactions = [];
 
   const end = Date.now();
   const time = end - start;
   logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
 
-  return { blocks, clusters: cpfpClusters };
+  return { blocks, rates, clusters: cpfpClusters };
 }
 
 // traverse in-mempool ancestors
 // recursion unavoidable, but should be limited to depth < 25 by mempool policy
 function setRelatives(
   tx: AuditTransaction,
-  mempool: { [txid: string]: AuditTransaction },
+  mempool: Map<number, AuditTransaction>,
 ): void {
-  for (const parent of tx.vin) {
-    const parentTx = mempool[parent];
+  for (const parent of tx.inputs) {
+    const parentTx = mempool.get(parent);
     if (parentTx && !tx.ancestorMap?.has(parent)) {
       tx.ancestorMap.set(parent, parentTx);
       parentTx.children.add(tx);
@@ -227,7 +232,7 @@ function setRelatives(
         setRelatives(parentTx, mempool);
       }
       parentTx.ancestorMap.forEach((ancestor) => {
-        tx.ancestorMap.set(ancestor.txid, ancestor);
+        tx.ancestorMap.set(ancestor.uid, ancestor);
       });
     }
   };
@@ -245,7 +250,7 @@ function setRelatives(
 // avoids recursion to limit call stack depth
 function updateDescendants(
   rootTx: AuditTransaction,
-  mempool: { [txid: string]: AuditTransaction },
+  mempool: Map<number, AuditTransaction>,
   modified: PairingHeap<AuditTransaction>,
 ): void {
   const descendantSet: Set<AuditTransaction> = new Set();
@@ -261,9 +266,9 @@ function updateDescendants(
   });
   while (descendants.length) {
     descendantTx = descendants.pop();
-    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.uid)) {
       // remove tx as ancestor
-      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorMap.delete(rootTx.uid);
       descendantTx.ancestorFee -= rootTx.fee;
       descendantTx.ancestorWeight -= rootTx.weight;
       tmpScore = descendantTx.score;

backend/src/api/websocket-handler.ts

@@ -247,14 +247,14 @@ class WebsocketHandler {
     });
   }
 
-  async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+  async $handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
     newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
+      await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true);
     } else {
       mempoolBlocks.updateMempoolBlocks(newMempool, true);
     }
@@ -425,13 +425,19 @@ class WebsocketHandler {
 
     if (config.MEMPOOL.AUDIT) {
       let projectedBlocks;
+      let auditMempool = _memPool;
       // template calculation functions have mempool side effects, so calculate audits using
       // a cloned copy of the mempool if we're running a different algorithm for mempool updates
-      const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
-      if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
-        projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
+      const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
+      if (separateAudit) {
+        auditMempool = deepClone(_memPool);
+        if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+          projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false);
+        } else {
+          projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
+        }
       } else {
-        projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
+        projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
       }
 
       if (Common.indexingEnabled() && memPool.isInSync()) {
@@ -477,16 +483,14 @@ class WebsocketHandler {
       }
     }
 
-    const removed: string[] = [];
     // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {
       delete _memPool[txId];
-      removed.push(txId);
       rbfCache.evict(txId);
     }
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true);
+      await mempoolBlocks.$makeBlockTemplates(_memPool, true);
     } else {
       mempoolBlocks.updateMempoolBlocks(_memPool, true);
     }

backend/src/index.ts

@@ -120,7 +120,7 @@ class Server {
     await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
     if (config.MEMPOOL.ENABLED) {
-      diskCache.loadMempoolCache();
+      await diskCache.$loadMempoolCache();
     }
 
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
@@ -178,8 +178,8 @@ class Server {
           logger.debug(msg);
         }
       }
-      memPool.deleteExpiredTransactions();
       await blocks.$updateBlocks();
+      memPool.deleteExpiredTransactions();
       await memPool.$updateMempool();
       indexer.$run();
 
@@ -206,6 +206,8 @@ class Server {
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
       this.currentBackendRetryInterval *= 2;
       this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
+    } finally {
+      diskCache.unlock();
     }
   }
 
@@ -238,7 +240,7 @@ class Server {
     websocketHandler.setupConnectionHandling();
     if (config.MEMPOOL.ENABLED) {
       statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
-      memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler));
       blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
     }
     priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));

backend/src/mempool.interfaces.ts

@@ -80,17 +80,22 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   bestDescendant?: BestDescendant | null;
   cpfpChecked?: boolean;
   deleteAfter?: number;
+  position?: {
+    block: number,
+    vsize: number,
+  };
+  uid?: number;
 }
 
 export interface AuditTransaction {
-  txid: string;
+  uid: number;
   fee: number;
   weight: number;
   feePerVsize: number;
   effectiveFeePerVsize: number;
-  vin: string[];
+  inputs: number[];
   relativesSet: boolean;
-  ancestorMap: Map<string, AuditTransaction>;
+  ancestorMap: Map<number, AuditTransaction>;
   children: Set<AuditTransaction>;
   ancestorFee: number;
   ancestorWeight: number;
@@ -100,13 +105,25 @@ export interface AuditTransaction {
   modifiedNode: HeapNode<AuditTransaction>;
 }
 
+export interface CompactThreadTransaction {
+  uid: number;
+  fee: number;
+  weight: number;
+  feePerVsize: number;
+  effectiveFeePerVsize?: number;
+  inputs: number[];
+  cpfpRoot?: string;
+  cpfpChecked?: boolean;
+  dirty?: boolean;
+}
+
 export interface ThreadTransaction {
   txid: string;
   fee: number;
   weight: number;
   feePerVsize: number;
   effectiveFeePerVsize?: number;
-  vin: string[];
+  inputs: number[];
   cpfpRoot?: string;
   cpfpChecked?: boolean;
 }

backend/src/repositories/BlocksRepository.ts

@@ -466,30 +466,6 @@ class BlocksRepository {
     }
   }
 
-  /**
-   * Get one block by hash
-   */
-  public async $getBlockByHash(hash: string): Promise<object | null> {
-    try {
-      const query = `
-        SELECT ${BLOCK_DB_FIELDS}
-        FROM blocks
-        JOIN pools ON blocks.pool_id = pools.id
-        WHERE hash = ?;
-      `;
-      const [rows]: any[] = await DB.query(query, [hash]);
-
-      if (rows.length <= 0) {
-        return null;
-      }
- 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
-    } catch (e) {
-      logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   /**
    * Return blocks difficulty
    */
@@ -599,7 +575,6 @@ class BlocksRepository {
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
           logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
-          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
           await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
           return false;
@@ -619,7 +594,7 @@ class BlocksRepository {
    * Delete blocks from the database from blockHeight
    */
   public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks from height ${blockHeight} from the database`);
+    logger.info(`Delete newer blocks from height ${blockHeight} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
@@ -978,6 +953,7 @@ class BlocksRepository {
     }
 
     // If we're missing block summary related field, check if we can populate them on the fly now
+    // This is for example triggered upon re-org
     if (Common.blocksSummariesIndexingEnabled() &&
       (extras.medianFeeAmt === null || extras.feePercentiles === null))
     {
@@ -985,7 +961,7 @@ class BlocksRepository {
       if (extras.feePercentiles === null) {
         const block = await bitcoinClient.getBlock(dbBlk.id, 2);
         const summary = blocks.summarizeBlock(block);
-        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+        await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.hash, summary.transactions);
         extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
       }
       if (extras.feePercentiles !== null) {

backend/src/repositories/BlocksSummariesRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { BlockSummary } from '../mempool.interfaces';
+import { BlockSummary, TransactionStripped } from '../mempool.interfaces';
 
 class BlocksSummariesRepository {
   public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
@@ -17,7 +17,7 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
+  public async $saveSummary(params: { height: number, mined?: BlockSummary}): Promise<void> {
     const blockId = params.mined?.id;
     try {
       const transactions = JSON.stringify(params.mined?.transactions || []);
@@ -37,6 +37,20 @@ class BlocksSummariesRepository {
     }
   }
 
+  public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
+    try {
+      const transactionsStr = JSON.stringify(transactions);
+      await DB.query(`
+        INSERT INTO blocks_summaries
+        SET height = ?, transactions = ?, id = ?
+        ON DUPLICATE KEY UPDATE transactions = ?`,
+        [blockHeight, transactionsStr, blockId, transactionsStr]);
+    } catch (e: any) {
+      logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
   public async $saveTemplate(params: { height: number, template: BlockSummary}) {
     const blockId = params.template?.id;
     try {
@@ -68,19 +82,6 @@ class BlocksSummariesRepository {
     return [];
   }
 
-  /**
-   * Delete blocks from the database from blockHeight
-   */
-  public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
-
-    try {
-      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
-    } catch (e) {
-      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
   /**
    * Get the fee percentiles if the block has already been indexed, [] otherwise
    * 

backend/src/repositories/HashratesRepository.ts

@@ -220,7 +220,7 @@ class HashratesRepository {
    * Delete hashrates from the database from timestamp
    */
   public async $deleteHashratesFromTimestamp(timestamp: number) {
-    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`);
+    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);

frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html

@@ -14,7 +14,7 @@
           <a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height }}</a>
         </td>
         <td class="date text-left">
-          <app-time kind="since" [time]="diffChange.timestamp" [fastRender]="true"></app-time>
+          <app-time kind="since" [time]="diffChange.timestamp" [fastRender]="true" [precision]="1"></app-time>
         </td>
         <td class="text-right">{{ diffChange.difficultyShorten }}</td>
         <td class="text-right" [style]="diffChange.change >= 0 ? 'color: #42B747' : 'color: #B74242'">

frontend/src/app/components/difficulty-mining/difficulty-mining.component.html

@@ -10,7 +10,7 @@
             <ng-template #blocksPlural let-i i18n="shared.blocks">{{ i }} <span class="shared-block">blocks</span></ng-template>
             <ng-template #blocksSingular let-i i18n="shared.block">{{ i }} <span class="shared-block">block</span></ng-template>
           </div>
-          <div class="symbol"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true"></app-time></div>
+          <div class="symbol"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true" [precision]="1"></app-time></div>
         </div>
         <div class="item">
           <h5 class="card-title" i18n="difficulty-box.estimate">Estimate</h5>
@@ -54,7 +54,7 @@
             <ng-template #blocksPlural let-i i18n="shared.blocks">{{ i }} <span class="shared-block">blocks</span></ng-template>
             <ng-template #blocksSingular let-i i18n="shared.block">{{ i }} <span class="shared-block">block</span></ng-template>
           </div>
-          <div class="symbol"><app-time kind="until" [time]="epochData.timeUntilHalving" [fastRender]="true"></app-time></div>
+          <div class="symbol"><app-time kind="until" [time]="epochData.timeUntilHalving" [fastRender]="true" [precision]="1"></app-time></div>
         </div>
       </div>
     </div>

frontend/src/app/components/difficulty/difficulty.component.html

@@ -37,7 +37,7 @@
         <div class="difficulty-stats">
           <div class="item">
             <div class="card-text">
-              ~<app-time [time]="epochData.timeAvg / 1000" [forceFloorOnTimeIntervals]="['minute']" [fractionDigits]="1"></app-time>
+              ~<app-time [time]="epochData.timeAvg / 1000" [fractionDigits]="1"></app-time>
             </div>
             <div class="symbol" i18n="difficulty-box.average-block-time">Average block time</div>
           </div>
@@ -68,7 +68,7 @@
             </div>
           </div>
           <div class="item">
-            <div class="card-text"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true"></app-time></div>
+            <div class="card-text"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true" [precision]="1"></app-time></div>
             <div class="symbol">
               {{ epochData.retargetDateString }}
             </div>

frontend/src/app/components/fees-box/fees-box.component.html

@@ -1,4 +1,4 @@
-<div class="fee-estimation-wrapper" *ngIf="(isLoadingWebSocket$ | async) === false && (recommendedFees$ | async) as recommendedFees; else loadingFees">
+<div class="fee-estimation-wrapper" *ngIf="(isLoading$ | async) === false && (recommendedFees$ | async) as recommendedFees; else loadingFees">
   <div class="d-flex">
     <div class="fee-progress-bar" [style.background]="noPriority">
       <span class="fee-label" i18n="fees-box.no-priority" i18n-ngbTooltip="Transaction feerate tooltip (economy)" ngbTooltip="Either 2x the minimum, or the Low Priority rate (whichever is lower)" placement="top">No Priority</span>

frontend/src/app/components/fees-box/fees-box.component.ts

@@ -1,9 +1,9 @@
 import { Component, OnInit, ChangeDetectionStrategy } from '@angular/core';
 import { StateService } from '../../services/state.service';
-import { Observable } from 'rxjs';
+import { Observable, combineLatest } from 'rxjs';
 import { Recommendedfees } from '../../interfaces/websocket.interface';
 import { feeLevels, mempoolFeeColors } from '../../app.constants';
-import { tap } from 'rxjs/operators';
+import { map, startWith, tap } from 'rxjs/operators';
 
 @Component({
   selector: 'app-fees-box',
@@ -12,7 +12,7 @@ import { tap } from 'rxjs/operators';
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class FeesBoxComponent implements OnInit {
-  isLoadingWebSocket$: Observable<boolean>;
+  isLoading$: Observable<boolean>;
   recommendedFees$: Observable<Recommendedfees>;
   gradient = 'linear-gradient(to right, #2e324e, #2e324e)';
   noPriority = '#2e324e';
@@ -22,7 +22,12 @@ export class FeesBoxComponent implements OnInit {
   ) { }
 
   ngOnInit(): void {
-    this.isLoadingWebSocket$ = this.stateService.isLoadingWebSocket$;
+    this.isLoading$ = combineLatest(
+      this.stateService.isLoadingWebSocket$.pipe(startWith(false)),
+      this.stateService.loadingIndicators$.pipe(startWith({ mempool: 0 })),
+    ).pipe(map(([socket, indicators]) => {
+      return socket || (indicators.mempool != null && indicators.mempool !== 100);
+    }));
     this.recommendedFees$ = this.stateService.recommendedFees$
       .pipe(
         tap((fees) => {

frontend/src/app/components/mempool-blocks/mempool-blocks.component.html

@@ -26,7 +26,7 @@
                 <app-time kind="until" [time]="(1 * i) + now + 61000" [fastRender]="false" [fixedRender]="true"></app-time>
               </ng-template>
               <ng-template #timeDiffMainnet>
-                <app-time kind="until" [time]="da.timeAvg * (i + 1) + now + da.timeOffset" [fastRender]="false" [fixedRender]="true" [forceFloorOnTimeIntervals]="['hour']"></app-time>
+                <app-time kind="until" [time]="da.timeAvg * (i + 1) + now + da.timeOffset" [fastRender]="false" [fixedRender]="true"></app-time>
               </ng-template>
             </div>
             <ng-template #mergedBlock>

frontend/src/app/components/search-form/search-form.component.ts

@@ -153,7 +153,7 @@ export class SearchFormComponent implements OnInit {
           const matchesBlockHeight = this.regexBlockheight.test(searchText);
           const matchesTxId = this.regexTransaction.test(searchText) && !this.regexBlockhash.test(searchText);
           const matchesBlockHash = this.regexBlockhash.test(searchText);
-          const matchesAddress = this.regexAddress.test(searchText);
+          const matchesAddress = !matchesTxId && this.regexAddress.test(searchText);
 
           if (matchesAddress && this.network === 'bisq') {
             searchText = 'B' + searchText;
@@ -198,7 +198,7 @@ export class SearchFormComponent implements OnInit {
     const searchText = result || this.searchForm.value.searchText.trim();
     if (searchText) {
       this.isSearching = true;
-      if (this.regexAddress.test(searchText)) {
+      if (!this.regexTransaction.test(searchText) && this.regexAddress.test(searchText)) {
         this.navigate('/address/', searchText);
       } else if (this.regexBlockhash.test(searchText) || this.regexBlockheight.test(searchText)) {
         this.navigate('/block/', searchText);

frontend/src/app/components/time/time.component.ts

@@ -10,6 +10,16 @@ import { dates } from '../../shared/i18n/dates';
 export class TimeComponent implements OnInit, OnChanges, OnDestroy {
   interval: number;
   text: string;
+  units: string[] = ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'];
+  precisionThresholds = {
+    year: 100,
+    month: 18,
+    week: 12,
+    day: 31,
+    hour: 48,
+    minute: 90,
+    second: 90
+  };
   intervals = {};
 
   @Input() time: number;
@@ -18,7 +28,7 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
   @Input() fastRender = false;
   @Input() fixedRender = false;
   @Input() relative = false;
-  @Input() forceFloorOnTimeIntervals: string[];
+  @Input() precision: number = 0;
   @Input() fractionDigits: number = 0;
 
   constructor(
@@ -83,23 +93,24 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
     }
 
     let counter: number;
-    for (const i in this.intervals) {
-      if (this.kind !== 'until' || this.forceFloorOnTimeIntervals && this.forceFloorOnTimeIntervals.indexOf(i) > -1) {
-        counter = Math.floor(seconds / this.intervals[i]);
-      } else {
-        counter = Math.round(seconds / this.intervals[i]);
+    for (const [index, unit] of this.units.entries()) {
+      let precisionUnit = this.units[Math.min(this.units.length - 1, index + this.precision)];
+      counter = Math.floor(seconds / this.intervals[unit]);
+      const precisionCounter = Math.floor(seconds / this.intervals[precisionUnit]);
+      if (precisionCounter > this.precisionThresholds[precisionUnit]) {
+        precisionUnit = unit;
       }
-      let rounded = counter;
-      if (this.fractionDigits) {
-        const roundFactor = Math.pow(10,this.fractionDigits);
-        rounded = Math.round((seconds / this.intervals[i]) * roundFactor) / roundFactor;
-      }
-      const dateStrings = dates(rounded);
       if (counter > 0) {
+        let rounded = Math.round(seconds / this.intervals[precisionUnit]);
+        if (this.fractionDigits) {
+          const roundFactor = Math.pow(10,this.fractionDigits);
+          rounded = Math.round((seconds / this.intervals[precisionUnit]) * roundFactor) / roundFactor;
+        }
+        const dateStrings = dates(rounded);
         switch (this.kind) {
           case 'since':
-            if (counter === 1) {
-              switch (i) { // singular (1 day)
+            if (rounded === 1) {
+              switch (precisionUnit) { // singular (1 day)
                 case 'year': return $localize`:@@time-since:${dateStrings.i18nYear}:DATE: ago`; break;
                 case 'month': return $localize`:@@time-since:${dateStrings.i18nMonth}:DATE: ago`; break;
                 case 'week': return $localize`:@@time-since:${dateStrings.i18nWeek}:DATE: ago`; break;
@@ -109,7 +120,7 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
                 case 'second': return $localize`:@@time-since:${dateStrings.i18nSecond}:DATE: ago`; break;
               }
             } else {
-              switch (i) { // plural (2 days)
+              switch (precisionUnit) { // plural (2 days)
                 case 'year': return $localize`:@@time-since:${dateStrings.i18nYears}:DATE: ago`; break;
                 case 'month': return $localize`:@@time-since:${dateStrings.i18nMonths}:DATE: ago`; break;
                 case 'week': return $localize`:@@time-since:${dateStrings.i18nWeeks}:DATE: ago`; break;
@@ -121,8 +132,8 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
             }
             break;
           case 'until':
-            if (counter === 1) {
-              switch (i) { // singular (In ~1 day)
+            if (rounded === 1) {
+              switch (precisionUnit) { // singular (In ~1 day)
                 case 'year': return $localize`:@@time-until:In ~${dateStrings.i18nYear}:DATE:`; break;
                 case 'month': return $localize`:@@time-until:In ~${dateStrings.i18nMonth}:DATE:`; break;
                 case 'week': return $localize`:@@time-until:In ~${dateStrings.i18nWeek}:DATE:`; break;
@@ -132,7 +143,7 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
                 case 'second': return $localize`:@@time-until:In ~${dateStrings.i18nSecond}:DATE:`;
               }
             } else {
-              switch (i) { // plural (In ~2 days)
+              switch (precisionUnit) { // plural (In ~2 days)
                 case 'year': return $localize`:@@time-until:In ~${dateStrings.i18nYears}:DATE:`; break;
                 case 'month': return $localize`:@@time-until:In ~${dateStrings.i18nMonths}:DATE:`; break;
                 case 'week': return $localize`:@@time-until:In ~${dateStrings.i18nWeeks}:DATE:`; break;
@@ -144,8 +155,8 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
             }
             break;
           case 'span':
-            if (counter === 1) {
-              switch (i) { // singular (1 day)
+            if (rounded === 1) {
+              switch (precisionUnit) { // singular (1 day)
                 case 'year': return $localize`:@@time-span:After ${dateStrings.i18nYear}:DATE:`; break;
                 case 'month': return $localize`:@@time-span:After ${dateStrings.i18nMonth}:DATE:`; break;
                 case 'week': return $localize`:@@time-span:After ${dateStrings.i18nWeek}:DATE:`; break;
@@ -155,7 +166,7 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
                 case 'second': return $localize`:@@time-span:After ${dateStrings.i18nSecond}:DATE:`; break;
               }
             } else {
-              switch (i) { // plural (2 days)
+              switch (precisionUnit) { // plural (2 days)
                 case 'year': return $localize`:@@time-span:After ${dateStrings.i18nYears}:DATE:`; break;
                 case 'month': return $localize`:@@time-span:After ${dateStrings.i18nMonths}:DATE:`; break;
                 case 'week': return $localize`:@@time-span:After ${dateStrings.i18nWeeks}:DATE:`; break;
@@ -167,8 +178,8 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
             }
             break;
           default:
-            if (counter === 1) {
-              switch (i) { // singular (1 day)
+            if (rounded === 1) {
+              switch (precisionUnit) { // singular (1 day)
                 case 'year': return dateStrings.i18nYear; break;
                 case 'month': return dateStrings.i18nMonth; break;
                 case 'week': return dateStrings.i18nWeek; break;
@@ -178,7 +189,7 @@ export class TimeComponent implements OnInit, OnChanges, OnDestroy {
                 case 'second': return dateStrings.i18nSecond; break;
               }
             } else {
-              switch (i) { // plural (2 days)
+              switch (precisionUnit) { // plural (2 days)
                 case 'year': return dateStrings.i18nYears; break;
                 case 'month': return dateStrings.i18nMonths; break;
                 case 'week': return dateStrings.i18nWeeks; break;

frontend/src/app/components/transaction/transaction.component.html

@@ -119,7 +119,7 @@
                           <app-time kind="until" [time]="(60 * 1000 * txInBlockIndex) + now" [fastRender]="false" [fixedRender]="true"></app-time>
                         </ng-template>
                         <ng-template #timeEstimateDefault>
-                          <app-time kind="until" *ngIf="(timeAvg$ | async) as timeAvg;" [time]="(timeAvg * txInBlockIndex) + now + timeAvg" [fastRender]="false" [fixedRender]="true" [forceFloorOnTimeIntervals]="['hour']"></app-time>
+                          <app-time kind="until" *ngIf="(timeAvg$ | async) as timeAvg;" [time]="(timeAvg * txInBlockIndex) + now + timeAvg" [fastRender]="false" [fixedRender]="true"></app-time>
                         </ng-template>
                       </ng-template>
                     </ng-template>

frontend/src/app/components/transaction/transaction.component.ts

@@ -49,6 +49,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
   blocksSubscription: Subscription;
   queryParamsSubscription: Subscription;
   urlFragmentSubscription: Subscription;
+  mempoolBlocksSubscription: Subscription;
   fragmentParams: URLSearchParams;
   rbfTransaction: undefined | Transaction;
   replaced: boolean = false;
@@ -59,7 +60,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
   fetchRbfHistory$ = new Subject<string>();
   fetchCachedTx$ = new Subject<string>();
   isCached: boolean = false;
-  now = new Date().getTime();
+  now = Date.now();
   timeAvg$: Observable<number>;
   liquidUnblinding = new LiquidUnblinding();
   inputIndex: number;
@@ -308,7 +309,6 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
           this.isLoadingTx = false;
           this.error = undefined;
           this.waitingForTransaction = false;
-          this.setMempoolBlocksSubscription();
           this.websocketService.startTrackTransaction(tx.txid);
           this.graphExpanded = false;
           this.setupGraph();
@@ -391,6 +391,34 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       this.setFlowEnabled();
       this.setGraphSize();
     });
+
+    this.mempoolBlocksSubscription = this.stateService.mempoolBlocks$.subscribe((mempoolBlocks) => {
+      if (!this.tx) {
+        return;
+      }
+
+      this.now = Date.now();
+
+      const txFeePerVSize =
+        this.tx.effectiveFeePerVsize || this.tx.fee / (this.tx.weight / 4);
+
+      let found = false;
+      this.txInBlockIndex = 0;
+      for (const block of mempoolBlocks) {
+        for (let i = 0; i < block.feeRange.length - 1 && !found; i++) {
+          if (
+            txFeePerVSize <= block.feeRange[i + 1] &&
+            txFeePerVSize >= block.feeRange[i]
+          ) {
+            this.txInBlockIndex = mempoolBlocks.indexOf(block);
+            found = true;
+          }
+        }
+      }
+      if (!found && txFeePerVSize < mempoolBlocks[mempoolBlocks.length - 1].feeRange[0]) {
+        this.txInBlockIndex = 7;
+      }
+    });
   }
 
   ngAfterViewInit(): void {
@@ -407,28 +435,6 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     return of(false);
   }
 
-  setMempoolBlocksSubscription() {
-    this.stateService.mempoolBlocks$.subscribe((mempoolBlocks) => {
-      if (!this.tx) {
-        return;
-      }
-
-      const txFeePerVSize =
-        this.tx.effectiveFeePerVsize || this.tx.fee / (this.tx.weight / 4);
-
-      for (const block of mempoolBlocks) {
-        for (let i = 0; i < block.feeRange.length - 1; i++) {
-          if (
-            txFeePerVSize <= block.feeRange[i + 1] &&
-            txFeePerVSize >= block.feeRange[i]
-          ) {
-            this.txInBlockIndex = mempoolBlocks.indexOf(block);
-          }
-        }
-      }
-    });
-  }
-
   getTransactionTime() {
     this.apiService
       .getTransactionTimes$([this.tx.txid])
@@ -536,6 +542,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     this.queryParamsSubscription.unsubscribe();
     this.flowPrefSubscription.unsubscribe();
     this.urlFragmentSubscription.unsubscribe();
+    this.mempoolBlocksSubscription.unsubscribe();
     this.leaveTransaction();
   }
 }