machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/fiat-selector

Browse Source
This commit is contained in:
wiz
2023-02-15 12:10:07 +09:00
committed by GitHub
parent 368041e7d4 14a794d9cc
commit e009c78c3e
19 changed files with 257 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
backend/src/api/mempool-blocks.ts
View File

@@ -33,7 +33,7 @@ class MempoolBlocks {
return this.mempoolBlockDeltas;
}
public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
const latestMempool = memPool;
const memPoolArray: TransactionExtended[] = [];
for (const i in latestMempool) {
@@ -75,10 +75,14 @@ class MempoolBlocks {
logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
this.mempoolBlocks = blocks;
this.mempoolBlockDeltas = deltas;
if (saveResults) {
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
this.mempoolBlocks = blocks;
this.mempoolBlockDeltas = deltas;
}
return blocks;
}
private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
@@ -143,7 +147,7 @@ class MempoolBlocks {
return mempoolBlockDeltas;
}
public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }): Promise<void> {
public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
const strippedMempool: { [txid: string]: ThreadTransaction } = {};
@@ -184,19 +188,21 @@ class MempoolBlocks {
this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
const { blocks, clusters } = await workerResultPromise;
this.processBlockTemplates(newMempool, blocks, clusters);
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
return this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
} catch (e) {
logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
}
return this.mempoolBlocks;
}
public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[]): Promise<void> {
public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
if (!this.txSelectionWorker) {
// need to reset the worker
return this.makeBlockTemplates(newMempool);
this.makeBlockTemplates(newMempool, saveResults);
return;
}
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
@@ -224,16 +230,16 @@ class MempoolBlocks {
this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
const { blocks, clusters } = await workerResultPromise;
this.processBlockTemplates(newMempool, blocks, clusters);
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
} catch (e) {
logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
}
}
private processBlockTemplates(mempool, blocks, clusters): void {
private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] {
// update this thread's mempool with the results
blocks.forEach(block => {
block.forEach(tx => {
@@ -278,10 +284,13 @@ class MempoolBlocks {
}).filter(tx => !!tx), undefined, undefined, blockIndex);
});
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
if (saveResults) {
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
this.mempoolBlocks = mempoolBlocks;
this.mempoolBlockDeltas = deltas;
}
this.mempoolBlocks = mempoolBlocks;
this.mempoolBlockDeltas = deltas;
return mempoolBlocks;
}
private dataToMempoolBlocks(transactions: TransactionExtended[],

21
backend/src/api/websocket-handler.ts
View File

@@ -19,6 +19,7 @@ import feeApi from './fee-api';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import Audit from './audit';
import { deepClone } from '../utils/clone';
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
@@ -251,9 +252,9 @@ class WebsocketHandler {
}
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid));
await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
} else {
mempoolBlocks.updateMempoolBlocks(newMempool);
mempoolBlocks.updateMempoolBlocks(newMempool, true);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
@@ -418,16 +419,18 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
let projectedBlocks;
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
await mempoolBlocks.makeBlockTemplates(_memPool);
projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
}
if (Common.indexingEnabled() && memPool.isInSync()) {
const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
@@ -471,9 +474,9 @@ class WebsocketHandler {
}
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.updateBlockTemplates(_memPool, [], removed);
await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);
mempoolBlocks.updateMempoolBlocks(_memPool, true);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();

46
backend/src/repositories/CpfpRepository.ts
View File

@@ -111,7 +111,7 @@ class CpfpRepository {
}
}
public async $getCluster(clusterRoot: string): Promise<Cluster> {
public async $getCluster(clusterRoot: string): Promise<Cluster | void> {
const [clusterRows]: any = await DB.query(
`
SELECT *
@@ -121,8 +121,11 @@ class CpfpRepository {
[clusterRoot]
);
const cluster = clusterRows[0];
cluster.txs = this.unpack(cluster.txs);
return cluster;
if (cluster?.txs) {
cluster.txs = this.unpack(cluster.txs);
return cluster;
}
return;
}
public async $deleteClustersFrom(height: number): Promise<void> {
@@ -136,9 +139,9 @@ class CpfpRepository {
[height]
) as RowDataPacket[][];
if (rows?.length) {
for (let clusterToDelete of rows) {
const txs = this.unpack(clusterToDelete.txs);
for (let tx of txs) {
for (const clusterToDelete of rows) {
const txs = this.unpack(clusterToDelete?.txs);
for (const tx of txs) {
await transactionRepository.$removeTransaction(tx.txid);
}
}
@@ -204,20 +207,25 @@ class CpfpRepository {
return [];
}
const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
const txs: Ancestor[] = [];
const view = new DataView(arrayBuffer);
for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
const weight = view.getUint32(offset + 32);
const fee = Number(view.getBigUint64(offset + 36));
txs.push({
txid,
weight,
fee
});
try {
const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
const txs: Ancestor[] = [];
const view = new DataView(arrayBuffer);
for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
const weight = view.getUint32(offset + 32);
const fee = Number(view.getBigUint64(offset + 36));
txs.push({
txid,
weight,
fee
});
}
return txs;
} catch (e) {
logger.warn(`Failed to unpack CPFP cluster. Reason: ` + (e instanceof Error ? e.message : e));
return [];
}
return txs;
}
}

34
backend/src/repositories/TransactionRepository.ts
View File

@@ -3,15 +3,6 @@ import logger from '../logger';
import { Ancestor, CpfpInfo } from '../mempool.interfaces';
import cpfpRepository from './CpfpRepository';
interface CpfpSummary {
txid: string;
cluster: string;
root: string;
txs: Ancestor[];
height: number;
fee_rate: number;
}
class TransactionRepository {
public async $setCluster(txid: string, clusterRoot: string): Promise<void> {
try {
@@ -72,7 +63,9 @@ class TransactionRepository {
const txid = txRows[0].id.toLowerCase();
const clusterId = txRows[0].root.toLowerCase();
const cluster = await cpfpRepository.$getCluster(clusterId);
return this.convertCpfp(txid, cluster);
if (cluster) {
return this.convertCpfp(txid, cluster);
}
}
} catch (e) {
logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
@@ -81,13 +74,18 @@ class TransactionRepository {
}
public async $removeTransaction(txid: string): Promise<void> {
await DB.query(
`
DELETE FROM compact_transactions
WHERE txid = UNHEX(?)
`,
[txid]
);
try {
await DB.query(
`
DELETE FROM compact_transactions
WHERE txid = UNHEX(?)
`,
[txid]
);
} catch (e) {
logger.warn('Cannot delete transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
private convertCpfp(txid, cluster): CpfpInfo {
@@ -95,7 +93,7 @@ class TransactionRepository {
const ancestors: Ancestor[] = [];
let matched = false;
for (const tx of cluster.txs) {
for (const tx of (cluster?.txs || [])) {
if (tx.txid === txid) {
matched = true;
} else if (!matched) {

14
backend/src/utils/clone.ts Normal file
View File

@@ -0,0 +1,14 @@
// simple recursive deep clone for literal-type objects
// does not preserve Dates, Maps, Sets etc
// does not support recursive objects
// properties deeper than maxDepth will be shallow cloned
export function deepClone(obj: any, maxDepth: number = 50, depth: number = 0): any {
let cloned = obj;
if (depth < maxDepth && typeof obj === 'object') {
cloned = Array.isArray(obj) ? [] : {};
for (const key in obj) {
cloned[key] = deepClone(obj[key], maxDepth, depth + 1);
}
}
return cloned;
}