Store block first seen in db

backend/src/api/audit.ts

@@ -1,4 +1,3 @@
-import * as fs from 'fs';
 import config from '../config';
 import logger from '../logger';
 import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
@@ -8,10 +7,10 @@ import transactionUtils from './transaction-utils';
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
-  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, hash: string)
-   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number, firstSeen: string | undefined } {
+  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
+   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1, firstSeen: undefined };
+      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
@@ -177,8 +176,6 @@ class Audit {
     }
     const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
 
-    const firstSeen = this.getFirstSeenFromLogs(hash);
-
     return {
       unseen,
       censored: Object.keys(isCensored),
@@ -190,39 +187,8 @@ class Audit {
       accelerated,
       score,
       similarity,
-      firstSeen
     };
   }
-
-  getFirstSeenFromLogs(hash: string): string | undefined {
-    const debugLogPath = config.CORE_RPC.DEBUG_LOG_PATH;
-    if (debugLogPath) {
-      try {
-        const fileDescriptor = fs.openSync(debugLogPath, 'r');
-        const bufferSize = 2048; // Read the last few lines of the file
-        const buffer = Buffer.alloc(bufferSize);
-        const fileSize = fs.statSync(debugLogPath).size;
-        const chunkSize = Math.min(bufferSize, fileSize);
-        fs.readSync(fileDescriptor, buffer, 0, chunkSize, fileSize - chunkSize);
-        const lines = buffer.toString('utf8', 0, chunkSize).split('\n');
-        fs.closeSync(fileDescriptor);
-  
-        for (let i = lines.length - 1; i >= 0; i--) {
-          const line = lines[i];
-          if (line && line.includes(`Saw new header hash=${hash}`)) {
-            // Extract time from log: "2021-08-31T12:34:56Z" or "2021-08-31T12:34:56.123456Z" if logtimemicros=1
-            const dateMatch = line.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.?\d{6})?Z/);
-            if (dateMatch) {
-              return dateMatch[0].replace("T", " ").replace("Z", "");
-            }
-          }
-        }
-      } catch (e) {
-        logger.err(`Cannot parse block first seen time from Core logs. Reason: ` + (e instanceof Error ? e.message : e));
-      }
-    }
-    return undefined;
-  }
 }
 
 export default new Audit();

backend/src/api/database-migration.ts

@@ -707,7 +707,7 @@ class DatabaseMigration {
     }
 
     if (databaseSchemaVersion < 83 && isBitcoin === true) {
-      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD first_seen timestamp(6) DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `blocks` ADD first_seen datetime(6) DEFAULT NULL');
       await this.updateToSchemaVersion(83);
     }
   }

backend/src/api/websocket-handler.ts

@@ -16,6 +16,7 @@ import transactionUtils from './transaction-utils';
 import rbfCache, { ReplacementInfo } from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
+import BlocksRepository from '../repositories/BlocksRepository';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import Audit from './audit';
@@ -34,6 +35,7 @@ interface AddressTransactions {
 }
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import { calculateMempoolTxCpfp } from './cpfp';
+import { getRecentFirstSeen } from '../utils/file-read';
 
 // valid 'want' subscriptions
 const wantable = [
@@ -975,7 +977,7 @@ class WebsocketHandler {
       }
 
       if (Common.indexingEnabled()) {
-        const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity, firstSeen } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool, block.id);
+        const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
         const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -1012,7 +1014,6 @@ class WebsocketHandler {
           matchRate: matchRate,
           expectedFees: totalFees,
           expectedWeight: totalWeight,
-          firstSeen: firstSeen,
         });
 
         if (block.extras) {
@@ -1029,6 +1030,14 @@ class WebsocketHandler {
       }
     }
 
+    if (config.CORE_RPC.DEBUG_LOG_PATH && block.extras) {
+      const firstSeen = getRecentFirstSeen(block.id);
+      if (firstSeen) {
+        BlocksRepository.$saveFirstSeenTime(block.id, firstSeen);
+        block.extras.firstSeen = firstSeen;
+      }
+    }
+
     const confirmedTxids: { [txid: string]: boolean } = {};
 
     // Update mempool to remove transactions included in the new block