machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Configurable esplora batch sizes

Browse Source
This commit is contained in:
Mononaut
2023-11-15 06:10:00 +00:00
parent 1aea3fcac5
commit 35d7c55c1d
11 changed files with 30 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -53,6 +53,7 @@
"ESPLORA": {
"REST_API_URL": "__ESPLORA_REST_API_URL__",
"UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
"BATCH_QUERY_BASE_SIZE": 1000,
"RETRY_UNIX_SOCKET_AFTER": 888,
"REQUEST_TIMEOUT": 10000,
"FALLBACK_TIMEOUT": 5000,
@@ -140,6 +141,7 @@
},
"REDIS": {
"ENABLED": false,
"UNIX_SOCKET_PATH": "/tmp/redis.sock"
"UNIX_SOCKET_PATH": "/tmp/redis.sock",
"BATCH_QUERY_BASE_SIZE": 5000
}
}

4
backend/src/__tests__/config.test.ts
View File

@@ -55,6 +55,7 @@ describe('Mempool Backend Config', () => {
expect(config.ESPLORA).toStrictEqual({
REST_API_URL: 'http://127.0.0.1:3000',
UNIX_SOCKET_PATH: null,
BATCH_QUERY_BASE_SIZE: 1000,
RETRY_UNIX_SOCKET_AFTER: 30000,
REQUEST_TIMEOUT: 10000,
FALLBACK_TIMEOUT: 5000,
@@ -144,7 +145,8 @@ describe('Mempool Backend Config', () => {
expect(config.REDIS).toStrictEqual({
ENABLED: false,
UNIX_SOCKET_PATH: ''
UNIX_SOCKET_PATH: '',
BATCH_QUERY_BASE_SIZE: 5000,
});
});
});

2
backend/src/api/mempool.ts
View File

@@ -235,7 +235,7 @@ class Mempool {
if (!loaded) {
const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]);
const sliceLength = 10000;
const sliceLength = config.ESPLORA.BATCH_QUERY_BASE_SIZE;
for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) {
const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength);
const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false);

2
backend/src/api/rbf-cache.ts
View File

@@ -480,7 +480,7 @@ class RbfCache {
};
if (config.MEMPOOL.BACKEND === 'esplora') {
const sliceLength = 250;
const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 40);
for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) {
const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength);
try {

5
backend/src/api/redis-cache.ts
View File

@@ -122,8 +122,9 @@ class RedisCache {
async $removeTransactions(transactions: string[]) {
try {
await this.$ensureConnected();
for (let i = 0; i < Math.ceil(transactions.length / 10000); i++) {
const slice = transactions.slice(i * 10000, (i + 1) * 10000);
const sliceLength = config.REDIS.BATCH_QUERY_BASE_SIZE;
for (let i = 0; i < Math.ceil(transactions.length / sliceLength); i++) {
const slice = transactions.slice(i * sliceLength, (i + 1) * sliceLength);
await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`));
logger.debug(`Deleted ${slice.length} transactions from the Redis cache`);
}

4
backend/src/config.ts
View File

@@ -43,6 +43,7 @@ interface IConfig {
ESPLORA: {
REST_API_URL: string;
UNIX_SOCKET_PATH: string | void | null;
BATCH_QUERY_BASE_SIZE: number;
RETRY_UNIX_SOCKET_AFTER: number;
REQUEST_TIMEOUT: number;
FALLBACK_TIMEOUT: number;
@@ -151,6 +152,7 @@ interface IConfig {
REDIS: {
ENABLED: boolean;
UNIX_SOCKET_PATH: string;
BATCH_QUERY_BASE_SIZE: number;
},
}
@@ -195,6 +197,7 @@ const defaults: IConfig = {
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',
'UNIX_SOCKET_PATH': null,
'BATCH_QUERY_BASE_SIZE': 1000,
'RETRY_UNIX_SOCKET_AFTER': 30000,
'REQUEST_TIMEOUT': 10000,
'FALLBACK_TIMEOUT': 5000,
@@ -303,6 +306,7 @@ const defaults: IConfig = {
'REDIS': {
'ENABLED': false,
'UNIX_SOCKET_PATH': '',
'BATCH_QUERY_BASE_SIZE': 5000,
},
};

2
backend/src/tasks/lightning/forensics.service.ts
View File

@@ -79,7 +79,7 @@ class ForensicsService {
}
let progress = 0;
const sliceLength = 1000;
const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 10);
// process batches of 1000 channels
for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) {
const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength);

2
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -290,7 +290,7 @@ class NetworkSyncService {
const allChannels = await channelsApi.$getChannelsByStatus([0, 1]);
const sliceLength = 5000;
const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 2);
// process batches of 5000 channels
for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) {
const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength);