CPFP support (#395)

* CPFP support.

fixes #5
fixes #353
fixes #360

* Use effectiveFeePerVsize for mempool statistics.

* Renaming endpoint cpfp-info to just cpfp.

* Renaming decended to BestDescendant.

* Updating language file with new strings.

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -3,7 +3,6 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
-  $getRawTransactionBitcoind(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
   $getBlockHeightTip(): Promise<number>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;

backend/src/api/bitcoin/bitcoin-api.ts

@@ -22,16 +22,6 @@ class BitcoinApi implements AbstractBitcoinApi {
     });
   }
 
-  $getRawTransactionBitcoind(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
-    return this.bitcoindClient.getRawTransaction(txId, true)
-      .then((transaction: IBitcoinApi.Transaction) => {
-        if (skipConversion) {
-          return transaction;
-        }
-        return this.$convertTransaction(transaction, addPrevout);
-      });
-  }
-
   $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
     const txInMempool = mempool.getMempool()[txId];
@@ -47,6 +37,9 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getRawTransaction(txId, true)
       .then((transaction: IBitcoinApi.Transaction) => {
         if (skipConversion) {
+          transaction.vout.forEach((vout) => {
+            vout.value = vout.value * 100000000;
+          });
           return transaction;
         }
         return this.$convertTransaction(transaction, addPrevout);

backend/src/api/bitcoin/esplora-api.ts

@@ -48,11 +48,6 @@ class ElectrsApi implements AbstractBitcoinApi {
     throw new Error('Method getAddressTransactions not implemented.');
   }
 
-  $getRawTransactionBitcoind(txId: string): Promise<IEsploraApi.Transaction> {
-    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
-      .then((response) => response.data);
-  }
-
   $getAddressPrefix(prefix: string): string[] {
     throw new Error('Method not implemented.');
   }

backend/src/api/blocks.ts

@@ -84,14 +84,21 @@ class Blocks {
         }
       }
 
+      transactions.forEach((tx) => {
+        if (!tx.cpfpChecked) {
+          Common.setRelativesAndGetCpfpInfo(tx, mempool);
+        }
+      });
+
       logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`);
 
       const blockExtended: BlockExtended = Object.assign({}, block);
       blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
       blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-      transactions.sort((a, b) => b.feePerVsize - a.feePerVsize);
-      blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0;
-      blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0];
+      transactions.shift();
+      transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+      blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0;
+      blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions, 8) : [0, 0];
 
       if (block.height % 2016 === 0) {
         this.lastDifficultyAdjustmentTime = block.timestamp;

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 
 export class Common {
   static median(numbers: number[]) {
@@ -20,16 +20,16 @@ export class Common {
   }
 
   static getFeesInRange(transactions: TransactionExtended[], rangeLength: number) {
-    const arr = [transactions[transactions.length - 1].feePerVsize];
+    const arr = [transactions[transactions.length - 1].effectiveFeePerVsize];
     const chunk = 1 / (rangeLength - 1);
     let itemsToAdd = rangeLength - 2;
 
     while (itemsToAdd > 0) {
-      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].feePerVsize);
+      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].effectiveFeePerVsize);
       itemsToAdd--;
     }
 
-    arr.push(transactions[0].feePerVsize);
+    arr.push(transactions[0].effectiveFeePerVsize);
     return arr;
   }
 
@@ -71,4 +71,63 @@ export class Common {
        }, ms);
     });
   }
+
+  static setRelativesAndGetCpfpInfo(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): CpfpInfo {
+    const parents = this.findAllParents(tx, memPool);
+
+    let totalWeight = tx.weight + parents.reduce((prev, val) => prev + val.weight, 0);
+    let totalFees = tx.fee + parents.reduce((prev, val) => prev + val.fee, 0);
+
+    tx.ancestors = parents
+      .map((t) => {
+        return {
+          txid: t.txid,
+          weight: t.weight,
+          fee: t.fee,
+        };
+      });
+
+    // Add high (high fee) decendant weight and fees
+    if (tx.bestDescendant) {
+      totalWeight += tx.bestDescendant.weight;
+      totalFees += tx.bestDescendant.fee;
+    }
+
+    tx.effectiveFeePerVsize = totalFees / (totalWeight / 4);
+    tx.cpfpChecked = true;
+
+    return {
+      ancestors: tx.ancestors,
+      bestDescendant: tx.bestDescendant || null,
+    };
+  }
+
+
+  private static findAllParents(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): TransactionExtended[] {
+    let parents: TransactionExtended[] = [];
+    tx.vin.forEach((parent) => {
+      const parentTx = memPool[parent.txid];
+      if (parentTx) {
+        if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.feePerVsize) {
+          if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) {
+            parentTx.bestDescendant = {
+              weight: tx.weight + tx.bestDescendant.weight,
+              fee: tx.fee + tx.bestDescendant.fee,
+              txid: tx.txid,
+            };
+          }
+        } else if (tx.feePerVsize > parentTx.feePerVsize) {
+          parentTx.bestDescendant = {
+            weight: tx.weight,
+            fee: tx.fee,
+            txid: tx.txid
+          };
+        }
+        parents.push(parentTx);
+        parents = parents.concat(this.findAllParents(parentTx, memPool));
+      }
+    });
+    return parents;
+  }
+
 }

backend/src/api/mempool-blocks.ts

@@ -1,3 +1,4 @@
+import logger from '../logger';
 import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
@@ -33,9 +34,40 @@ class MempoolBlocks {
         memPoolArray.push(latestMempool[i]);
       }
     }
+    const start = new Date().getTime();
+
+    // Clear bestDescendants & ancestors
+    memPoolArray.forEach((tx) => {
+      tx.bestDescendant = null;
+      tx.ancestors = [];
+      tx.cpfpChecked = false;
+      if (!tx.effectiveFeePerVsize) {
+        tx.effectiveFeePerVsize = tx.feePerVsize;
+      }
+    });
+
+    // First sort
     memPoolArray.sort((a, b) => b.feePerVsize - a.feePerVsize);
-    const transactionsSorted = memPoolArray.filter((tx) => tx.feePerVsize);
-    this.mempoolBlocks = this.calculateMempoolBlocks(transactionsSorted);
+
+    // Loop through and traverse all ancestors and sum up all the sizes + fees
+    // Pass down size + fee to all unconfirmed children
+    let sizes = 0;
+    memPoolArray.forEach((tx, i) => {
+      sizes += tx.weight
+      if (sizes > 4000000 * 8) {
+        return;
+      }
+      Common.setRelativesAndGetCpfpInfo(tx, memPool);
+    });
+
+    // Final sort, by effective fee
+    memPoolArray.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+
+    const end = new Date().getTime();
+    const time = end - start;
+    logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
+
+    this.mempoolBlocks = this.calculateMempoolBlocks(memPoolArray);
   }
 
   private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
@@ -77,7 +109,7 @@ class MempoolBlocks {
       blockVSize: blockVSize,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
-      medianFee: Common.percentile(transactions.map((tx) => tx.feePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
     };

backend/src/api/statistics.ts

@@ -68,13 +68,13 @@ class Statistics {
       }
     }
     // Remove 0 and undefined
-    memPoolArray = memPoolArray.filter((tx) => tx.feePerVsize);
+    memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
 
     if (!memPoolArray.length) {
       return;
     }
 
-    memPoolArray.sort((a, b) => a.feePerVsize - b.feePerVsize);
+    memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize);
     const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
     const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
 
@@ -85,7 +85,7 @@ class Statistics {
 
     memPoolArray.forEach((transaction) => {
       for (let i = 0; i < logFees.length; i++) {
-        if ((logFees[i] === 2000 && transaction.feePerVsize >= 2000) || transaction.feePerVsize <= logFees[i]) {
+        if ((logFees[i] === 2000 && transaction.effectiveFeePerVsize >= 2000) || transaction.effectiveFeePerVsize <= logFees[i]) {
           if (weightVsizeFees[logFees[i]]) {
             weightVsizeFees[logFees[i]] += transaction.vsize;
           } else {

backend/src/api/transaction-utils.ts

@@ -26,9 +26,11 @@ class TransactionUtils {
   }
 
   private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
+    const feePerVbytes = Math.max(1, (transaction.fee || 0) / (transaction.weight / 4));
     const transactionExtended: TransactionExtended = Object.assign({
       vsize: Math.round(transaction.weight / 4),
-      feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)),
+      feePerVsize: feePerVbytes,
+      effectiveFeePerVsize: feePerVbytes,
     }, transaction);
     if (!transaction.status.confirmed) {
       transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));

backend/src/index.ts

@@ -73,6 +73,8 @@ class Server {
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
 
+    this.setUpWebsocketHandling();
+
     diskCache.loadMempoolCache();
 
     if (config.DATABASE.ENABLED) {
@@ -86,7 +88,6 @@ class Server {
     fiatConversion.startService();
 
     this.setUpHttpApiRoutes();
-    this.setUpWebsocketHandling();
     this.runMainUpdateLoop();
 
     if (config.BISQ_BLOCKS.ENABLED) {
@@ -145,6 +146,7 @@ class Server {
   setUpHttpApiRoutes() {
     this.app
       .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', routes.getCpfpInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', routes.getMempoolBlocks)
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', routes.getBackendInfo)

backend/src/mempool.interfaces.ts

@@ -26,6 +26,27 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   vsize: number;
   feePerVsize: number;
   firstSeen?: number;
+  effectiveFeePerVsize: number;
+  ancestors?: Ancestor[];
+  bestDescendant?: BestDescendant | null;
+  cpfpChecked?: boolean;
+}
+
+interface Ancestor {
+  txid: string;
+  weight: number;
+  fee: number;
+}
+
+interface BestDescendant {
+  txid: string;
+  weight: number;
+  fee: number;
+}
+
+export interface CpfpInfo {
+  ancestors: Ancestor[];
+  bestDescendant: BestDescendant | null;
 }
 
 export interface TransactionStripped {

backend/src/routes.ts

@@ -94,6 +94,30 @@ class Routes {
     res.json(times);
   }
 
+  public getCpfpInfo(req: Request, res: Response) {
+    if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
+      res.status(501).send(`Invalid transaction ID.`);
+      return;
+    }
+
+    const tx = mempool.getMempool()[req.params.txId];
+    if (!tx) {
+      res.status(404).send(`Transaction doesn't exist in the mempool.`);
+      return;
+    }
+
+    if (tx.cpfpChecked) {
+      res.json({
+        ancestors: tx.ancestors,
+        bestDescendant: tx.bestDescendant || null,
+      });
+    }
+
+    const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+
+    res.json(cpfpInfo);
+  }
+
   public getBackendInfo(req: Request, res: Response) {
     res.json(backendInfo.getBackendInfo());
   }