Merge branch 'master' into mononaut/fix-mined-rbf-conflicts

backend/src/api/audit.ts

@@ -1,19 +1,21 @@
 import config from '../config';
 import logger from '../logger';
 import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import rbfCache from './rbf-cache';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
   auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
-   : { censored: string[], added: string[], fresh: string[], sigop: string[], score: number, similarity: number } {
+   : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { censored: [], added: [], fresh: [], sigop: [], score: 0, similarity: 1 };
+      return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
     const added: string[] = []; // present in mined block, not in template
     const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement
     const isCensored = {}; // missing, without excuse
     const isDisplaced = {};
     let displacedWeight = 0;
@@ -35,7 +37,9 @@ class Audit {
     for (const txid of projectedBlocks[0].transactionIds) {
       if (!inBlock[txid]) {
         // tx is recent, may have reached the miner too late for inclusion
-        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+        if (rbfCache.isFullRbf(txid)) {
+          fullrbf.push(txid);
+        } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
           fresh.push(txid);
         } else {
           isCensored[txid] = true;
@@ -91,7 +95,9 @@ class Audit {
       if (inTemplate[tx.txid]) {
         matches.push(tx.txid);
       } else {
-        if (!isDisplaced[tx.txid]) {
+        if (rbfCache.isFullRbf(tx.txid)) {
+          fullrbf.push(tx.txid);
+        } else if (!isDisplaced[tx.txid]) {
           added.push(tx.txid);
         }
         overflowWeight += tx.weight;
@@ -138,6 +144,7 @@ class Audit {
       added,
       fresh,
       sigop: [],
+      fullrbf,
       score,
       similarity,
     };

backend/src/api/bitcoin/bitcoin-api.ts

@@ -29,6 +29,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       weight: block.weight,
       previousblockhash: block.previousblockhash,
       mediantime: block.mediantime,
+      stale: block.confirmations === -1,
     };
   }
 

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -89,6 +89,7 @@ export namespace IEsploraApi {
     weight: number;
     previousblockhash: string;
     mediantime: number;
+    stale: boolean;
   }
 
   export interface Address {

backend/src/api/blocks.ts

@@ -656,10 +656,6 @@ class Blocks {
       const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
       this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
-      // start async callbacks
-      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
-      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
-
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
@@ -671,9 +667,11 @@ class Blocks {
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
+            this.blocks = this.blocks.slice(0, -10);
             this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
+              this.blocks.push(newBlock);
               this.updateTimerProgress(timer, `reindexed block`);
               let cpfpSummary;
               if (config.MEMPOOL.CPFP_INDEXING) {
@@ -722,6 +720,10 @@ class Blocks {
         }
       }
 
+      // start async callbacks
+      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
+      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
+
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
           await DifficultyAdjustmentsRepository.$saveAdjustments({
@@ -814,6 +816,16 @@ class Blocks {
     return blockExtended;
   }
 
+  public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
+    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
+    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
+    const blockExtended = await this.$getBlockExtended(block, transactions);
+
+    blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
+
+    return blockExtended;
+  }
+
   /**
    * Get one block by its hash
    */
@@ -831,7 +843,11 @@ class Blocks {
 
     // Bitcoin network, add our custom data on top
     const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
-    return await this.$indexBlock(block.height);
+    if (block.stale) {
+      return await this.$indexStaleBlock(hash);
+    } else {
+      return await this.$indexBlock(block.height);
+    }
   }
 
   public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 62;
+  private static currentVersion = 63;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -539,6 +539,10 @@ class DatabaseMigration {
       await this.updateToSchemaVersion(62);
     }
 
+    if (databaseSchemaVersion < 63 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(63);
+    }
   }
 
   /**

backend/src/api/rbf-cache.ts

@@ -169,6 +169,19 @@ class RbfCache {
     }
   }
 
+  // is the transaction involved in a full rbf replacement?
+  public isFullRbf(txid: string): boolean {
+    const treeId = this.treeMap.get(txid);
+    if (!treeId) {
+      return false;
+    }
+    const tree = this.rbfTrees.get(treeId);
+    if (!tree) {
+      return false;
+    }
+    return tree?.fullRbf;
+  }
+
   private cleanup(): void {
     const now = Date.now();
     for (const txid of this.expiring.keys()) {

backend/src/api/websocket-handler.ts

@@ -583,6 +583,10 @@ class WebsocketHandler {
 
     const _memPool = memPool.getMempool();
 
+    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
+    memPool.handleMinedRbfTransactions(rbfTransactions);
+    memPool.removeFromSpendMap(transactions);
+
     if (config.MEMPOOL.AUDIT) {
       let projectedBlocks;
       let auditMempool = _memPool;
@@ -605,7 +609,7 @@ class WebsocketHandler {
       }
 
       if (Common.indexingEnabled() && memPool.isInSync()) {
-        const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+        const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
         const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -633,6 +637,7 @@ class WebsocketHandler {
           missingTxs: censored,
           freshTxs: fresh,
           sigopTxs: sigop,
+          fullrbfTxs: fullrbf,
           matchRate: matchRate,
           expectedFees: totalFees,
           expectedWeight: totalWeight,
@@ -652,10 +657,6 @@ class WebsocketHandler {
       }
     }
 
-    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
-    memPool.handleMinedRbfTransactions(rbfTransactions);
-    memPool.removeFromSpendMap(transactions);
-
     // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {
       delete _memPool[txId];