machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revamping configuration file.

Browse Source 
fixes #141
This commit is contained in:
softsimon
2020-10-19 11:57:02 +07:00
parent 4e4a6f6c59
commit e2a7683828
17 changed files with 218 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/src/api/bisq/bisq.ts
View File

@@ -1,4 +1,4 @@
const config = require('../../../mempool-config.json');
import config from '../../config';
import * as fs from 'fs';
import * as request from 'request';
import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
@@ -8,7 +8,7 @@ import { StaticPool } from 'node-worker-threads-pool';
import logger from '../../logger';
class Bisq {
private static BLOCKS_JSON_FILE_PATH = '/all/blocks.json';
private static BLOCKS_JSON_FILE_PATH = config.BISQ_BLOCKS.DATA_PATH + '/all/blocks.json';
private latestBlockHeight = 0;
private blocks: BisqBlock[] = [];
private transactions: BisqTransaction[] = [];
@@ -87,8 +87,8 @@ class Bisq {
}
private checkForBisqDataFolder() {
if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
return process.exit(1);
}
}
@@ -98,7 +98,7 @@ class Bisq {
this.topDirectoryWatcher.close();
}
let fsWait: NodeJS.Timeout | null = null;
this.topDirectoryWatcher = fs.watch(config.BISQ_BLOCKS_DATA_PATH, () => {
this.topDirectoryWatcher = fs.watch(config.BISQ_BLOCKS.DATA_PATH, () => {
if (fsWait) {
clearTimeout(fsWait);
}
@@ -120,13 +120,13 @@ class Bisq {
if (this.subdirectoryWatcher) {
this.subdirectoryWatcher.close();
}
if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
setTimeout(() => this.startSubDirectoryWatcher(), 180000);
return;
}
let fsWait: NodeJS.Timeout | null = null;
this.subdirectoryWatcher = fs.watch(config.BISQ_BLOCKS_DATA_PATH + '/all', () => {
this.subdirectoryWatcher = fs.watch(config.BISQ_BLOCKS.DATA_PATH + '/all', () => {
if (fsWait) {
clearTimeout(fsWait);
}
@@ -249,7 +249,7 @@ class Bisq {
this.blocks.reverse();
this.latestBlockHeight = data.chainHeight;
const time = new Date().getTime() - start;
logger.debug('Bisq dump processed in ' + time + ' ms');
logger.debug('Bisq dump processed in ' + time + ' ms (worker thread)');
} else {
throw new Error(`Bisq dump didn't contain any blocks`);
}
@@ -258,10 +258,10 @@ class Bisq {
private loadData(): Promise<string> {
return new Promise((resolve, reject) => {
if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) {
if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
}
fs.readFile(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
fs.readFile(Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
if (err) {
reject(err);
}

4
backend/src/api/bisq/markets.ts
View File

@@ -1,4 +1,4 @@
const config = require('../../../mempool-config.json');
import config from '../../config';
import * as fs from 'fs';
import { OffersData as OffersData, TradesData, Currency } from './interfaces';
import bisqMarket from './markets-api';
@@ -6,7 +6,7 @@ import logger from '../../logger';
class Bisq {
private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000;
private static MARKET_JSON_PATH = config.BISQ_MARKETS_DATA_PATH + '/btc_mainnet/db';
private static MARKET_JSON_PATH = config.BISQ_MARKETS.DATA_PATH + '/btc_mainnet/db';
private static MARKET_JSON_FILE_PATHS = {
cryptoCurrency: '/crypto_currency_list.json',
fiatCurrency: '/fiat_currency_list.json',

18
backend/src/api/bitcoin/electrs-api.ts
View File

@@ -1,4 +1,4 @@
const config = require('../../../mempool-config.json');
import config from '../../config';
import { Transaction, Block, MempoolInfo } from '../../interfaces';
import * as request from 'request';
@@ -9,7 +9,7 @@ class ElectrsApi {
getMempoolInfo(): Promise<MempoolInfo> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/mempool', { json: true, timeout: 10000 }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/mempool', { json: true, timeout: 10000 }, (err, res, response) => {
if (err) {
reject('getMempoolInfo error: ' + err.message || err);
} else if (res.statusCode !== 200) {
@@ -30,7 +30,7 @@ class ElectrsApi {
getRawMempool(): Promise<Transaction['txid'][]> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/mempool/txids', { json: true, timeout: 10000, forever: true }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/mempool/txids', { json: true, timeout: 10000, forever: true }, (err, res, response) => {
if (err) {
reject('getRawMempool error: ' + err.message || err);
} else if (res.statusCode !== 200) {
@@ -48,7 +48,7 @@ class ElectrsApi {
getRawTransaction(txId: string): Promise<Transaction> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/tx/' + txId, { json: true, timeout: 10000, forever: true }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/tx/' + txId, { json: true, timeout: 10000, forever: true }, (err, res, response) => {
if (err) {
reject('getRawTransaction error: ' + err.message || err);
} else if (res.statusCode !== 200) {
@@ -66,7 +66,7 @@ class ElectrsApi {
getBlockHeightTip(): Promise<number> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/blocks/tip/height', { json: true, timeout: 10000 }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/blocks/tip/height', { json: true, timeout: 10000 }, (err, res, response) => {
if (err) {
reject('getBlockHeightTip error: ' + err.message || err);
} else if (res.statusCode !== 200) {
@@ -80,7 +80,7 @@ class ElectrsApi {
getTxIdsForBlock(hash: string): Promise<string[]> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/block/' + hash + '/txids', { json: true, timeout: 10000 }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/block/' + hash + '/txids', { json: true, timeout: 10000 }, (err, res, response) => {
if (err) {
reject('getTxIdsForBlock error: ' + err.message || err);
} else if (res.statusCode !== 200) {
@@ -98,7 +98,7 @@ class ElectrsApi {
getBlockHash(height: number): Promise<string> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/block-height/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/block-height/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
if (err) {
reject('getBlockHash error: ' + err.message || err);
} else if (res.statusCode !== 200) {
@@ -112,7 +112,7 @@ class ElectrsApi {
getBlocksFromHeight(height: number): Promise<string> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/blocks/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/blocks/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
if (err) {
reject('getBlocksFromHeight error: ' + err.message || err);
} else if (res.statusCode !== 200) {
@@ -126,7 +126,7 @@ class ElectrsApi {
getBlock(hash: string): Promise<Block> {
return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/block/' + hash, { json: true, timeout: 10000 }, (err, res, response) => {
request(config.ELECTRS.REST_API_URL + '/block/' + hash, { json: true, timeout: 10000 }, (err, res, response) => {
if (err) {
reject('getBlock error: ' + err.message || err);
} else if (res.statusCode !== 200) {

13
backend/src/api/blocks.ts
View File

@@ -1,4 +1,3 @@
const config = require('../../mempool-config.json');
import bitcoinApi from './bitcoin/electrs-api';
import logger from '../logger';
import memPool from './mempool';
@@ -6,6 +5,8 @@ import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces'
import { Common } from './common';
class Blocks {
private static INITIAL_BLOCK_AMOUNT = 8;
private static KEEP_BLOCK_AMOUNT = 24;
private blocks: Block[] = [];
private currentBlockHeight = 0;
private lastDifficultyAdjustmentTime = 0;
@@ -29,14 +30,14 @@ class Blocks {
const blockHeightTip = await bitcoinApi.getBlockHeightTip();
if (this.blocks.length === 0) {
this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT;
this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT;
} else {
this.currentBlockHeight = this.blocks[this.blocks.length - 1].height;
}
if (blockHeightTip - this.currentBlockHeight > config.INITIAL_BLOCK_AMOUNT * 2) {
logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.INITIAL_BLOCK_AMOUNT} recent blocks`);
this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT;
if (blockHeightTip - this.currentBlockHeight > Blocks.INITIAL_BLOCK_AMOUNT * 2) {
logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${Blocks.INITIAL_BLOCK_AMOUNT} recent blocks`);
this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT;
}
if (!this.lastDifficultyAdjustmentTime) {
@@ -91,7 +92,7 @@ class Blocks {
}
this.blocks.push(block);
if (this.blocks.length > config.KEEP_BLOCK_AMOUNT) {
if (this.blocks.length > Blocks.KEEP_BLOCK_AMOUNT) {
this.blocks.shift();
}

4
backend/src/api/disk-cache.ts
View File

@@ -1,4 +1,4 @@
const config = require('../../mempool-config.json');
import config from '../config';
import * as fs from 'fs';
import * as process from 'process';
import memPool from './mempool';
@@ -9,7 +9,7 @@ class DiskCache {
static FILE_NAME = './cache.json';
constructor() {
if (process.env.workerId === '0' || !config.CLUSTER_NUM_CORES || config.CLUSTER_NUM_CORES === 1) {
if (process.env.workerId === '0' || !config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
process.on('SIGINT', () => {
this.saveCacheToDisk();
process.exit(2);

13
backend/src/api/donations.ts
View File

@@ -1,4 +1,4 @@
const config = require('../../mempool-config.json');
import config from '../config';
import * as request from 'request';
import { DB } from '../database';
import logger from '../logger';
@@ -6,16 +6,19 @@ import logger from '../logger';
class Donations {
private notifyDonationStatusCallback: ((invoiceId: string) => void) | undefined;
private options = {
baseUrl: config.BTCPAY_URL,
baseUrl: config.SPONSORS.BTCPAY_URL,
headers: {
'Content-Type': 'application/json',
'Authorization': config.BTCPAY_AUTH,
'Authorization': config.SPONSORS.BTCPAY_AUTH,
},
};
sponsorsCache: any[] = [];
constructor() {
if (!config.SPONSORS.ENABLED) {
return;
}
this.$updateCache();
}
@@ -39,7 +42,7 @@ class Donations {
'orderId': orderId,
'currency': 'BTC',
'itemDesc': 'Sponsor mempool.space',
'notificationUrl': config.BTCPAY_WEBHOOK_URL,
'notificationUrl': config.SPONSORS.BTCPAY_WEBHOOK_URL,
'redirectURL': 'https://mempool.space/about'
};
return new Promise((resolve, reject) => {
@@ -185,7 +188,7 @@ class Donations {
uri: `https://api.twitter.com/1.1/users/show.json?screen_name=${handle}`,
json: true,
headers: {
Authorization: 'Bearer ' + config.TWITTER_BEARER_AUTH
Authorization: 'Bearer ' + config.SPONSORS.TWITTER_BEARER_AUTH
},
}, (err, res, body) => {
if (err) { return reject(err); }

4
backend/src/api/fee-api.ts
View File

@@ -1,11 +1,11 @@
const config = require('../../mempool-config.json');
import config from '../config';
import { MempoolBlock } from '../interfaces';
import projectedBlocks from './mempool-blocks';
class FeeApi {
constructor() { }
defaultFee = config.NETWORK === 'liquid' ? 0.1 : 1;
defaultFee = config.MEMPOOL.NETWORK === 'liquid' ? 0.1 : 1;
public getRecommendedFee() {
const pBlocks = projectedBlocks.getMempoolBlocks();

4
backend/src/api/mempool-blocks.ts
View File

@@ -1,8 +1,8 @@
const config = require('../../mempool-config.json');
import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../interfaces';
import { Common } from './common';
class MempoolBlocks {
private static DEFAULT_PROJECTED_BLOCKS_AMOUNT = 8;
private mempoolBlocks: MempoolBlockWithTransactions[] = [];
constructor() {}
@@ -43,7 +43,7 @@ class MempoolBlocks {
let blockSize = 0;
let transactions: TransactionExtended[] = [];
transactionsSorted.forEach((tx) => {
if (blockVSize + tx.vsize <= 1000000 || mempoolBlocks.length === config.DEFAULT_PROJECTED_BLOCKS_AMOUNT - 1) {
if (blockVSize + tx.vsize <= 1000000 || mempoolBlocks.length === MempoolBlocks.DEFAULT_PROJECTED_BLOCKS_AMOUNT - 1) {
blockVSize += tx.vsize;
blockSize += tx.size;
transactions.push(tx);

13
backend/src/api/mempool.ts
View File

@@ -1,4 +1,4 @@
const config = require('../../mempool-config.json');
import config from '../config';
import bitcoinApi from './bitcoin/electrs-api';
import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond } from '../interfaces';
import logger from '../logger';
@@ -124,14 +124,13 @@ class Mempool {
}
}
if ((new Date().getTime()) - start > config.MEMPOOL_REFRESH_RATE_MS * 10) {
if ((new Date().getTime()) - start > config.MEMPOOL.WEBSOCKET_REFRESH_RATE_MS * 10) {
break;
}
}
// Prevent mempool from clear on bitcoind restart by delaying the deletion
if ((config.NETWORK === 'mainnet' || !config.NETWORK)
&& this.mempoolProtection === 0 && transactions.length / currentMempoolSize <= 0.80) {
if (config.MEMPOOL.NETWORK === 'mainnet' && this.mempoolProtection === 0 && transactions.length / currentMempoolSize <= 0.80) {
this.mempoolProtection = 1;
this.inSync = false;
logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`);
@@ -182,14 +181,14 @@ class Mempool {
}
private updateTxPerSecond() {
const nowMinusTimeSpan = new Date().getTime() - (1000 * config.TX_PER_SECOND_SPAN_SECONDS);
const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);
this.txPerSecond = this.txPerSecondArray.length / config.TX_PER_SECOND_SPAN_SECONDS || 0;
this.txPerSecond = this.txPerSecondArray.length / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD || 0;
this.vBytesPerSecondArray = this.vBytesPerSecondArray.filter((data) => data.unixTime > nowMinusTimeSpan);
if (this.vBytesPerSecondArray.length) {
this.vBytesPerSecond = Math.round(
this.vBytesPerSecondArray.map((data) => data.vSize).reduce((a, b) => a + b) / config.TX_PER_SECOND_SPAN_SECONDS
this.vBytesPerSecondArray.map((data) => data.vSize).reduce((a, b) => a + b) / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD
);
}
}

4
backend/src/api/websocket-handler.ts
View File

@@ -1,4 +1,3 @@
const config = require('../../mempool-config.json');
import logger from '../logger';
import * as WebSocket from 'ws';
import { Block, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic } from '../interfaces';
@@ -10,6 +9,7 @@ import fiatConversion from './fiat-conversion';
import { Common } from './common';
class WebsocketHandler {
private static INITIAL_BLOCK_AMOUNT = 8;
private wss: WebSocket.Server | undefined;
private nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
private extraInitProperties = {};
@@ -85,7 +85,7 @@ class WebsocketHandler {
'mempoolInfo': memPool.getMempoolInfo(),
'vBytesPerSecond': memPool.getVBytesPerSecond(),
'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
'blocks': _blocks.slice(Math.max(_blocks.length - config.INITIAL_BLOCK_AMOUNT, 0)),
'blocks': _blocks.slice(Math.max(_blocks.length - WebsocketHandler.INITIAL_BLOCK_AMOUNT, 0)),
'conversions': fiatConversion.getTickers()['BTCUSD'],
'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
'transactions': memPool.getLatestTransactions(),