Merge branch 'master' into junderw/psbt-complete-inputs

backend/src/api/audit.ts

@@ -0,0 +1,118 @@
+import logger from '../logger';
+import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+
+const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
+
+class Audit {
+  auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
+   : { censored: string[], added: string[], score: number } {
+    if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
+      return { censored: [], added: [], score: 0 };
+    }
+
+    const matches: string[] = []; // present in both mined block and template
+    const added: string[] = []; // present in mined block, not in template
+    const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const isCensored = {}; // missing, without excuse
+    const isDisplaced = {};
+    let displacedWeight = 0;
+
+    const inBlock = {};
+    const inTemplate = {};
+
+    const now = Math.round((Date.now() / 1000));
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+    // coinbase is always expected
+    if (transactions[0]) {
+      inTemplate[transactions[0].txid] = true;
+    }
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const txid of projectedBlocks[0].transactionIds) {
+      if (!inBlock[txid]) {
+        // tx is recent, may have reached the miner too late for inclusion
+        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          fresh.push(txid);
+        } else {
+          isCensored[txid] = true;
+        }
+        displacedWeight += mempool[txid].weight;
+      }
+      inTemplate[txid] = true;
+    }
+
+    displacedWeight += (4000 - transactions[0].weight);
+
+    logger.warn(`${fresh.length} fresh, ${Object.keys(isCensored).length} possibly censored, ${displacedWeight} displaced weight`);
+
+    // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
+    // these displaced transactions should occupy the first N weight units of the next projected block
+    let displacedWeightRemaining = displacedWeight;
+    let index = 0;
+    let lastFeeRate = Infinity;
+    let failures = 0;
+    while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
+      const txid = projectedBlocks[1].transactionIds[index];
+      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (fits || feeMatches) {
+        isDisplaced[txid] = true;
+        if (fits) {
+          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+        }
+        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+          displacedWeightRemaining -= mempool[txid].weight;
+        }
+        failures = 0;
+      } else {
+        failures++;
+      }
+      index++;
+    }
+
+    // mark unexpected transactions in the mined block as 'added'
+    let overflowWeight = 0;
+    for (const tx of transactions) {
+      if (inTemplate[tx.txid]) {
+        matches.push(tx.txid);
+      } else {
+        if (!isDisplaced[tx.txid]) {
+          added.push(tx.txid);
+        }
+        overflowWeight += tx.weight;
+      }
+    }
+
+    // transactions missing from near the end of our template are probably not being censored
+    let overflowWeightRemaining = overflowWeight;
+    let lastOverflowRate = 1.00;
+    index = projectedBlocks[0].transactionIds.length - 1;
+    while (index >= 0) {
+      const txid = projectedBlocks[0].transactionIds[index];
+      if (overflowWeightRemaining > 0) {
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+        lastOverflowRate = mempool[txid].effectiveFeePerVsize;
+      } else if (Math.floor(mempool[txid].effectiveFeePerVsize * 100) <= Math.ceil(lastOverflowRate * 100)) { // tolerance of 0.01 sat/vb
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+      }
+      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      index--;
+    }
+
+    const numCensored = Object.keys(isCensored).length;
+    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+
+    return {
+      censored: Object.keys(isCensored),
+      added,
+      score
+    };
+  }
+}
+
+export default new Audit();

backend/src/api/blocks.ts

@@ -20,6 +20,7 @@ import indexer from '../indexer';
 import fiatConversion from './fiat-conversion';
 import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
@@ -186,14 +187,18 @@ class Blocks {
       if (!pool) { // We should never have this situation in practise
         logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
-        return blockExtended;
+      } else {
+        blockExtended.extras.pool = {
+          id: pool.id,
+          name: pool.name,
+          slug: pool.slug,
+        };
       }
 
-      blockExtended.extras.pool = {
-        id: pool.id,
-        name: pool.name,
-        slug: pool.slug,
-      };
+      const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id);
+      if (auditSummary) {
+        blockExtended.extras.matchRate = auditSummary.matchRate;
+      }
     }
 
     return blockExtended;

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 40;
+  private static currentVersion = 41;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -348,6 +348,10 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
     }
+
+    if (databaseSchemaVersion < 41 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
+    }
   }
 
   /**

backend/src/api/explorer/nodes.api.ts

@@ -129,6 +129,56 @@ class NodesApi {
     }
   }
 
+  public async $getFeeHistogram(node_public_key: string): Promise<unknown> {
+    try {
+      const inQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node2_fee_rate WHEN node2_public_key = ? THEN node1_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [inRows]: any[] = await DB.query(inQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      const outQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node1_fee_rate WHEN node2_public_key = ? THEN node2_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [outRows]: any[] = await DB.query(outQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      return {
+        incoming: inRows.length > 0 ? inRows : [],
+        outgoing: outRows.length > 0 ? outRows : [],
+      };
+    } catch (e) {
+      logger.err(`Cannot get node fee distribution for ${node_public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
   public async $getAllNodes(): Promise<any> {
     try {
       const query = `SELECT * FROM nodes`;

backend/src/api/explorer/nodes.routes.ts

@@ -20,7 +20,9 @@ class NodesRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/fees/histogram', this.$getFeeHistogram)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
     ;
   }
 
@@ -33,6 +35,39 @@ class NodesRoutes {
     }
   }
 
+  private async $getNodeGroup(req: Request, res: Response) {
+    try {
+      let nodesList;
+      let nodes: any[] = [];
+      switch (config.MEMPOOL.NETWORK) {
+        case 'testnet':
+          nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
+          break;
+        case 'signet':
+          nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
+          break;
+        default:
+          nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
+      }
+
+      for (let pubKey of nodesList) {
+        try {
+          const node = await nodesApi.$getNode(pubKey);
+          if (node) {
+            nodes.push(node);
+          }
+        } catch (e) {}
+      }
+      
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNode(req: Request, res: Response) {
     try {
       const node = await nodesApi.$getNode(req.params.public_key);
@@ -61,6 +96,22 @@ class NodesRoutes {
     }
   }
 
+  private async $getFeeHistogram(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getFeeHistogram(req.params.public_key);
+      if (!node) {
+        res.status(404).send('Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNodesRanking(req: Request, res: Response): Promise<void> {
     try {
       const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);

backend/src/api/explorer/statistics.api.ts

@@ -6,7 +6,8 @@ class StatisticsApi {
   public async $getStatistics(interval: string | null = null): Promise<any> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity, tor_nodes, clearnet_nodes, unannounced_nodes
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
+      tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
       FROM lightning_stats`;
 
     if (interval) {

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -70,6 +70,8 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
       logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
       loggerTimer = new Date().getTime() / 1000;
     }
+
+    channelProcessed++;
   }
 
   return consolidatedChannelList;

backend/src/api/mempool-blocks.ts

@@ -1,7 +1,8 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, AuditTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
+import { PairingHeap } from '../utils/pairing-heap';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
@@ -72,6 +73,7 @@ class MempoolBlocks {
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
     const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+
     this.mempoolBlocks = blocks;
     this.mempoolBlockDeltas = deltas;
   }
@@ -99,6 +101,7 @@ class MempoolBlocks {
     if (transactions.length) {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
+
     // Calculate change from previous block states
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
@@ -132,12 +135,286 @@ class MempoolBlocks {
         removed
       });
     }
+
     return {
       blocks: mempoolBlocks,
       deltas: mempoolBlockDeltas
     };
   }
 
+  /*
+  * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+  * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+  *
+  * blockLimit: number of blocks to build in total.
+  * weightLimit: maximum weight of transactions to consider using the selection algorithm.
+  *              if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate 
+  * condenseRest: whether to ignore excess transactions or append them to the final block.
+  */
+  public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): MempoolBlockWithTransactions[] {
+    const start = Date.now();
+    const auditPool: { [txid: string]: AuditTransaction } = {};
+    const mempoolArray: AuditTransaction[] = [];
+    const restOfArray: TransactionExtended[] = [];
+    
+    let weight = 0;
+    const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity;
+    // grab the top feerate txs up to maxWeight
+    Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+      weight += tx.weight;
+      if (weight >= maxWeight) {
+        restOfArray.push(tx);
+        return;
+      }
+      // initializing everything up front helps V8 optimize property access later
+      auditPool[tx.txid] = {
+        txid: tx.txid,
+        fee: tx.fee,
+        size: tx.size,
+        weight: tx.weight,
+        feePerVsize: tx.feePerVsize,
+        vin: tx.vin,
+        relativesSet: false,
+        ancestorMap: new Map<string, AuditTransaction>(),
+        children: new Set<AuditTransaction>(),
+        ancestorFee: 0,
+        ancestorWeight: 0,
+        score: 0,
+        used: false,
+        modified: false,
+        modifiedNode: null,
+      }
+      mempoolArray.push(auditPool[tx.txid]);
+    })
+
+    // Build relatives graph & calculate ancestor scores
+    for (const tx of mempoolArray) {
+      if (!tx.relativesSet) {
+        this.setRelatives(tx, auditPool);
+      }
+    }
+
+    // Sort by descending ancestor score
+    mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+
+    // Build blocks by greedily choosing the highest feerate package
+    // (i.e. the package rooted in the transaction with the best ancestor score)
+    const blocks: MempoolBlockWithTransactions[] = [];
+    let blockWeight = 4000;
+    let blockSize = 0;
+    let transactions: AuditTransaction[] = [];
+    const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+    let overflow: AuditTransaction[] = [];
+    let failures = 0;
+    let top = 0;
+    while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) {
+      // skip invalid transactions
+      while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
+        top++;
+      }
+
+      // Select best next package
+      let nextTx;
+      const nextPoolTx = mempoolArray[top];
+      const nextModifiedTx = modified.peek();
+      if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+        nextTx = nextPoolTx;
+        top++;
+      } else {
+        modified.pop();
+        if (nextModifiedTx) {
+          nextTx = nextModifiedTx;
+          nextTx.modifiedNode = undefined;
+        }
+      }
+
+      if (nextTx && !nextTx?.used) {
+        // Check if the package fits into this block
+        if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+          blockWeight += nextTx.ancestorWeight;
+          const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
+          // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+          const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+          const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
+          sortedTxSet.forEach((ancestor, i, arr) => {
+            const mempoolTx = mempool[ancestor.txid];
+            if (ancestor && !ancestor?.used) {
+              ancestor.used = true;
+              // update original copy of this tx with effective fee rate & relatives data
+              mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+              mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
+                return {
+                  txid: a.txid,
+                  fee: a.fee,
+                  weight: a.weight,
+                }
+              })
+              if (i < arr.length - 1) {
+                mempoolTx.bestDescendant = {
+                  txid: arr[arr.length - 1].txid,
+                  fee: arr[arr.length - 1].fee,
+                  weight: arr[arr.length - 1].weight,
+                };
+              }
+              transactions.push(ancestor);
+              blockSize += ancestor.size;
+            }
+          });
+
+          // remove these as valid package ancestors for any descendants remaining in the mempool
+          if (sortedTxSet.length) {
+            sortedTxSet.forEach(tx => {
+              this.updateDescendants(tx, auditPool, modified);
+            });
+          }
+
+          failures = 0;
+        } else {
+          // hold this package in an overflow list while we check for smaller options
+          overflow.push(nextTx);
+          failures++;
+        }
+      }
+
+      // this block is full
+      const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+      if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) {
+        // construct this block
+        if (transactions.length) {
+          blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+        }
+        // reset for the next block
+        transactions = [];
+        blockSize = 0;
+        blockWeight = 4000;
+
+        // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+        for (const overflowTx of overflow.reverse()) {
+          if (overflowTx.modified) {
+            overflowTx.modifiedNode = modified.add(overflowTx);
+          } else {
+            top--;
+            mempoolArray[top] = overflowTx;
+          }
+        }
+        overflow = [];
+      }
+    }
+    if (condenseRest) {
+      // pack any leftover transactions into the last block
+      for (const tx of overflow) {
+        if (!tx || tx?.used) {
+          continue;
+        }
+        blockWeight += tx.weight;
+        blockSize += tx.size;
+        transactions.push(tx);
+        tx.used = true;
+      }
+      const blockTransactions = transactions.map(t => mempool[t.txid])
+      restOfArray.forEach(tx => {
+        blockWeight += tx.weight;
+        blockSize += tx.size;
+        blockTransactions.push(tx);
+      });
+      if (blockTransactions.length) {
+        blocks.push(this.dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length));
+      }
+      transactions = [];
+    } else if (transactions.length) {
+      blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+    }
+
+    const end = Date.now();
+    const time = end - start;
+    logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+    return blocks;
+  }
+
+  // traverse in-mempool ancestors
+  // recursion unavoidable, but should be limited to depth < 25 by mempool policy
+  public setRelatives(
+    tx: AuditTransaction,
+    mempool: { [txid: string]: AuditTransaction },
+  ): void {
+    for (const parent of tx.vin) {
+      const parentTx = mempool[parent.txid];
+      if (parentTx && !tx.ancestorMap!.has(parent.txid)) {
+        tx.ancestorMap.set(parent.txid, parentTx);
+        parentTx.children.add(tx);
+        // visit each node only once
+        if (!parentTx.relativesSet) {
+          this.setRelatives(parentTx, mempool);
+        }
+        parentTx.ancestorMap.forEach((ancestor) => {
+          tx.ancestorMap.set(ancestor.txid, ancestor);
+        });
+      }
+    };
+    tx.ancestorFee = tx.fee || 0;
+    tx.ancestorWeight = tx.weight || 0;
+    tx.ancestorMap.forEach((ancestor) => {
+      tx.ancestorFee += ancestor.fee;
+      tx.ancestorWeight += ancestor.weight;
+    });
+    tx.score = tx.ancestorFee / (tx.ancestorWeight || 1);
+    tx.relativesSet = true;
+  }
+
+  // iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+  // avoids recursion to limit call stack depth
+  private updateDescendants(
+    rootTx: AuditTransaction,
+    mempool: { [txid: string]: AuditTransaction },
+    modified: PairingHeap<AuditTransaction>,
+  ): void {
+    const descendantSet: Set<AuditTransaction> = new Set();
+    // stack of nodes left to visit
+    const descendants: AuditTransaction[] = [];
+    let descendantTx;
+    let ancestorIndex;
+    let tmpScore;
+    rootTx.children.forEach(childTx => {
+      if (!descendantSet.has(childTx)) {
+        descendants.push(childTx);
+        descendantSet.add(childTx);
+      }
+    });
+    while (descendants.length) {
+      descendantTx = descendants.pop();
+      if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+        // remove tx as ancestor
+        descendantTx.ancestorMap.delete(rootTx.txid);
+        descendantTx.ancestorFee -= rootTx.fee;
+        descendantTx.ancestorWeight -= rootTx.weight;
+        tmpScore = descendantTx.score;
+        descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorWeight;
+
+        if (!descendantTx.modifiedNode) {
+          descendantTx.modified = true;
+          descendantTx.modifiedNode = modified.add(descendantTx);
+        } else {
+          // rebalance modified heap if score has changed
+          if (descendantTx.score < tmpScore) {
+            modified.decreasePriority(descendantTx.modifiedNode);
+          } else if (descendantTx.score > tmpScore) {
+            modified.increasePriority(descendantTx.modifiedNode);
+          }
+        }
+
+        // add this node's children to the stack
+        descendantTx.children.forEach(childTx => {
+          // visit each node only once
+          if (!descendantSet.has(childTx)) {
+            descendants.push(childTx);
+            descendantSet.add(childTx);
+          }
+        });
+      }
+    }
+  }
+
   private dataToMempoolBlocks(transactions: TransactionExtended[],
     blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
     let rangeLength = 4;

backend/src/api/mining/mining-routes.ts

@@ -238,6 +238,12 @@ class MiningRoutes {
   public async $getBlockAudit(req: Request, res: Response) {
     try {
       const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
+
+      if (!audit) {
+        res.status(404).send(`This block has not been audited.`);
+        return;
+      }
+
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());

backend/src/api/websocket-handler.ts

@@ -18,6 +18,7 @@ import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import Audit from './audit';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -405,75 +406,63 @@ class WebsocketHandler {
     });
   }
 
-  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
+  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
     let mBlocks: undefined | MempoolBlock[];
     let mBlockDeltas: undefined | MempoolBlockDelta[];
-    let matchRate = 0;
+    let matchRate;
     const _memPool = memPool.getMempool();
-    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
 
-    if (_mempoolBlocks[0]) {
-      const matches: string[] = [];
-      const added: string[] = [];
-      const missing: string[] = [];
+    if (Common.indexingEnabled()) {
+      const mempoolCopy = cloneMempool(_memPool);
+      const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2);
 
-      for (const txId of txIds) {
-        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
-          matches.push(txId);
-        } else {
-          added.push(txId);
+      const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, mempoolCopy);
+      matchRate = Math.round(score * 100 * 100) / 100;
+
+      const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
+        return {
+          txid: tx.txid,
+          vsize: tx.vsize,
+          fee: tx.fee ? Math.round(tx.fee) : 0,
+          value: tx.value,
+        };
+      }) : [];
+
+      BlocksSummariesRepository.$saveSummary({
+        height: block.height,
+        template: {
+          id: block.id,
+          transactions: stripped
         }
-        delete _memPool[txId];
-      }
+      });
 
-      for (const txId of _mempoolBlocks[0].transactionIds) {
-        if (matches.includes(txId) || added.includes(txId)) {
-          continue;
-        }
-        missing.push(txId);
-      }
+      BlocksAuditsRepository.$saveAudit({
+        time: block.timestamp,
+        height: block.height,
+        hash: block.id,
+        addedTxs: added,
+        missingTxs: censored,
+        matchRate: matchRate,
+      });
 
-      matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
-      mempoolBlocks.updateMempoolBlocks(_memPool);
-      mBlocks = mempoolBlocks.getMempoolBlocks();
-      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
-
-      if (Common.indexingEnabled()) {
-        const stripped = _mempoolBlocks[0].transactions.map((tx) => {
-          return {
-            txid: tx.txid,
-            vsize: tx.vsize,
-            fee: tx.fee ? Math.round(tx.fee) : 0,
-            value: tx.value,
-          };
-        });  
-        BlocksSummariesRepository.$saveSummary({
-          height: block.height,
-          template: {
-            id: block.id,
-            transactions: stripped
-          }
-        });
-
-        BlocksAuditsRepository.$saveAudit({
-          time: block.timestamp,
-          height: block.height,
-          hash: block.id,
-          addedTxs: added,
-          missingTxs: missing,
-          matchRate: matchRate,
-        });
+      if (block.extras) {
+        block.extras.matchRate = matchRate;
       }
     }
 
-    if (block.extras) {
-      block.extras.matchRate = matchRate;
+    // Update mempool to remove transactions included in the new block
+    for (const txId of txIds) {
+      delete _memPool[txId];
     }
 
+    mempoolBlocks.updateMempoolBlocks(_memPool);
+    mBlocks = mempoolBlocks.getMempoolBlocks();
+    mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
     const da = difficultyAdjustment.getDifficultyAdjustment();
     const fees = feeApi.getRecommendedFee();
 
@@ -580,4 +569,14 @@ class WebsocketHandler {
   }
 }
 
+function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } {
+  const cloned = {};
+  Object.keys(mempool).forEach(id => {
+    cloned[id] = {
+      ...mempool[id]
+    };
+  });
+  return cloned;
+}
+
 export default new WebsocketHandler();