machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/node-sockets-lnd

Browse Source
This commit is contained in:
Felipe Knorr Kuhn
2023-02-19 13:56:48 -08:00
committed by GitHub
parent 19036a451f 4591be9f36
commit 3940b331ab
98 changed files with 3056 additions and 1688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend/README.md
View File

@@ -218,3 +218,21 @@ Generate block at regular interval (every 10 seconds in this example):
```
watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
```
### Re-index tables
You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
Usage:
```
npm run start --reindex=blocks,hashrates
```
Example output:
```
Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
```
Reference: https://github.com/mempool/mempool/pull/1269

2
backend/mempool-config.sample.json
View File

@@ -15,7 +15,6 @@
"MEMPOOL_BLOCKS_AMOUNT": 8,
"INDEXING_BLOCKS_AMOUNT": 11000,
"BLOCKS_SUMMARIES_INDEXING": false,
"PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [],
"EXTERNAL_MAX_RETRY": 1,
@@ -25,6 +24,7 @@
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"AUDIT": false,
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false

2
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -16,7 +16,6 @@
"BLOCK_WEIGHT_UNITS": 6,
"INITIAL_BLOCKS_AMOUNT": 7,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"PRICE_FEED_UPDATE_INTERVAL": 9,
"USE_SECOND_NODE_FOR_MINFEE": 10,
"EXTERNAL_ASSETS": 11,
"EXTERNAL_MAX_RETRY": 12,
@@ -26,6 +25,7 @@
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"AUDIT": "__MEMPOOL_AUDIT__",
"ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"

2
backend/src/__tests__/config.test.ts
View File

@@ -29,7 +29,6 @@ describe('Mempool Backend Config', () => {
INITIAL_BLOCKS_AMOUNT: 8,
MEMPOOL_BLOCKS_AMOUNT: 8,
INDEXING_BLOCKS_AMOUNT: 11000,
PRICE_FEED_UPDATE_INTERVAL: 600,
USE_SECOND_NODE_FOR_MINFEE: false,
EXTERNAL_ASSETS: [],
EXTERNAL_MAX_RETRY: 1,
@@ -38,6 +37,7 @@ describe('Mempool Backend Config', () => {
STDOUT_LOG_MIN_PRIORITY: 'debug',
POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
AUDIT: false,
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
CPFP_INDEXING: false,

29
backend/src/api/bitcoin/esplora-api.ts
View File

@@ -1,8 +1,13 @@
import config from '../../config';
import axios, { AxiosRequestConfig } from 'axios';
import http from 'http';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IEsploraApi } from './esplora-api.interface';
const axiosConnection = axios.create({
httpAgent: new http.Agent({ keepAlive: true })
});
class ElectrsApi implements AbstractBitcoinApi {
axiosConfig: AxiosRequestConfig = {
timeout: 10000,
@@ -11,52 +16,52 @@ class ElectrsApi implements AbstractBitcoinApi {
constructor() { }
$getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
.then((response) => response.data);
}
$getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
.then((response) => response.data);
}
$getTransactionHex(txId: string): Promise<string> {
return axios.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
.then((response) => response.data);
}
$getBlockHeightTip(): Promise<number> {
return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
.then((response) => response.data);
}
$getBlockHashTip(): Promise<string> {
return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
.then((response) => response.data);
}
$getTxIdsForBlock(hash: string): Promise<string[]> {
return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
.then((response) => response.data);
}
$getBlockHash(height: number): Promise<string> {
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
.then((response) => response.data);
}
$getBlockHeader(hash: string): Promise<string> {
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
.then((response) => response.data);
}
$getBlock(hash: string): Promise<IEsploraApi.Block> {
return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
.then((response) => response.data);
}
$getRawBlock(hash: string): Promise<Buffer> {
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
.then((response) => { return Buffer.from(response.data); });
}
@@ -77,12 +82,12 @@ class ElectrsApi implements AbstractBitcoinApi {
}
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
.then((response) => response.data);
}
$getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
.then((response) => response.data);
}

25
backend/src/api/blocks.ts
View File

@@ -17,7 +17,6 @@ import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
import fiatConversion from './fiat-conversion';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
@@ -170,7 +169,7 @@ class Blocks {
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
blockExtended.extras.usd = fiatConversion.getConversionRates().USD;
blockExtended.extras.usd = priceUpdater.latestPrices.USD;
if (block.height === 0) {
blockExtended.extras.medianFee = 0; // 50th percentiles
@@ -212,9 +211,11 @@ class Blocks {
};
}
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blockExtended.extras.matchRate = auditScore.matchRate;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blockExtended.extras.matchRate = auditScore.matchRate;
}
}
}
@@ -599,9 +600,11 @@ class Blocks {
* Index a block if it's missing from the database. Returns the block after indexing
*/
public async $indexBlock(height: number): Promise<BlockExtended> {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock != null) {
return prepareBlock(dbBlock);
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock !== null) {
return prepareBlock(dbBlock);
}
}
const blockHash = await bitcoinApi.$getBlockHash(height);
@@ -609,7 +612,9 @@ class Blocks {
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
await blocksRepository.$saveBlockInDatabase(blockExtended);
if (Common.indexingEnabled()) {
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return prepareBlock(blockExtended);
}
@@ -713,7 +718,7 @@ class Blocks {
block = await this.$indexBlock(currentHeight);
returnBlocks.push(block);
} else if (nextHash != null) {
block = prepareBlock(await bitcoinClient.getBlock(nextHash));
block = await this.$indexBlock(currentHeight);
nextHash = block.previousblockhash;
returnBlocks.push(block);
}

123
backend/src/api/fiat-conversion.ts
View File

@@ -1,123 +0,0 @@
import logger from '../logger';
import * as http from 'http';
import * as https from 'https';
import axios, { AxiosResponse } from 'axios';
import { IConversionRates } from '../mempool.interfaces';
import config from '../config';
import backendInfo from './backend-info';
import { SocksProxyAgent } from 'socks-proxy-agent';
class FiatConversion {
private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
private conversionRates: IConversionRates = {};
private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
public ratesInitialized = false; // If true, it means rates are ready for use
constructor() {
for (const fiat of this.debasingFiatCurrencies) {
this.conversionRates[fiat] = 0;
}
}
public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
this.ratesChangedCallback = fn;
}
public startService() {
const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
logger.info('Starting currency rates service');
if (config.SOCKS5PROXY.ENABLED) {
logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
} else {
logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
}
setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
this.updateCurrency();
}
public getConversionRates() {
return this.conversionRates;
}
private async updateCurrency(): Promise<void> {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpAgent?: http.Agent;
httpsAgent?: https.Agent;
}
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0;
while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try {
if (config.SOCKS5PROXY.ENABLED) {
let socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
}
// Handle proxy agent for onion addresses
if (isHTTP) {
axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
} else {
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
}
logger.debug('Querying currency rates service...');
const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
if (response.statusText === 'error' || !response.data) {
throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
}
for (const rate of response.data.data) {
if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
}
}
this.ratesInitialized = true;
logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.conversionRates);
}
break;
} catch (e) {
logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
retry++;
}
}
}
}
export default new FiatConversion();

2
backend/src/api/pools-parser.ts
View File

@@ -127,7 +127,7 @@ class PoolsParser {
if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
finalPoolDataUpdate.push(poolObj);
}
} else {
} else if (config.DATABASE.ENABLED) {
// Double check that if we're not just renaming a pool (same address same regex)
const [poolToRename]: any[] = await DB.query(`
SELECT * FROM pools

84
backend/src/api/websocket-handler.ts
View File

@@ -8,7 +8,6 @@ import blocks from './blocks';
import memPool from './mempool';
import backendInfo from './backend-info';
import mempoolBlocks from './mempool-blocks';
import fiatConversion from './fiat-conversion';
import { Common } from './common';
import loadingIndicators from './loading-indicators';
import config from '../config';
@@ -20,6 +19,7 @@ import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import Audit from './audit';
import { deepClone } from '../utils/clone';
import priceUpdater from '../tasks/price-updater';
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
@@ -214,7 +214,7 @@ class WebsocketHandler {
'mempoolInfo': memPool.getMempoolInfo(),
'vBytesPerSecond': memPool.getVBytesPerSecond(),
'blocks': _blocks,
'conversions': fiatConversion.getConversionRates(),
'conversions': priceUpdater.latestPrices,
'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
'transactions': memPool.getLatestTransactions(),
'backendInfo': backendInfo.getBackendInfo(),
@@ -419,49 +419,51 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
let projectedBlocks;
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
} else {
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
}
if (config.MEMPOOL.AUDIT) {
let projectedBlocks;
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
} else {
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
}
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
transactions: stripped
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
transactions: stripped
}
});
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: censored,
freshTxs: fresh,
matchRate: matchRate,
});
if (block.extras) {
block.extras.matchRate = matchRate;
}
});
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: censored,
freshTxs: fresh,
matchRate: matchRate,
});
if (block.extras) {
block.extras.matchRate = matchRate;
}
}

4
backend/src/config.ts
View File

@@ -19,7 +19,6 @@ interface IConfig {
MEMPOOL_BLOCKS_AMOUNT: number;
INDEXING_BLOCKS_AMOUNT: number;
BLOCKS_SUMMARIES_INDEXING: boolean;
PRICE_FEED_UPDATE_INTERVAL: number;
USE_SECOND_NODE_FOR_MINFEE: boolean;
EXTERNAL_ASSETS: string[];
EXTERNAL_MAX_RETRY: number;
@@ -29,6 +28,7 @@ interface IConfig {
AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
AUDIT: boolean;
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
CPFP_INDEXING: boolean;
@@ -140,7 +140,6 @@ const defaults: IConfig = {
'MEMPOOL_BLOCKS_AMOUNT': 8,
'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
'BLOCKS_SUMMARIES_INDEXING': false,
'PRICE_FEED_UPDATE_INTERVAL': 600,
'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [],
'EXTERNAL_MAX_RETRY': 1,
@@ -150,6 +149,7 @@ const defaults: IConfig = {
'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'AUDIT': false,
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'CPFP_INDEXING': false,

55
backend/src/index.ts
View File

@@ -10,7 +10,6 @@ import memPool from './api/mempool';
import diskCache from './api/disk-cache';
import statistics from './api/statistics/statistics';
import websocketHandler from './api/websocket-handler';
import fiatConversion from './api/fiat-conversion';
import bisq from './api/bisq/bisq';
import bisqMarkets from './api/bisq/markets';
import logger from './logger';
@@ -36,6 +35,8 @@ import liquidRoutes from './api/liquid/liquid.routes';
import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
import forensicsService from './tasks/lightning/forensics.service';
import priceUpdater from './tasks/price-updater';
import { AxiosError } from 'axios';
class Server {
private wss: WebSocket.Server | undefined;
@@ -78,25 +79,6 @@ class Server {
async startServer(worker = false): Promise<void> {
logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
this.app
.use((req: Request, res: Response, next: NextFunction) => {
res.setHeader('Access-Control-Allow-Origin', '*');
next();
})
.use(express.urlencoded({ extended: true }))
.use(express.text({ type: ['text/plain', 'application/base64'] }))
;
this.server = http.createServer(this.app);
this.wss = new WebSocket.Server({ server: this.server });
this.setUpWebsocketHandling();
await syncAssets.syncAssets$();
if (config.MEMPOOL.ENABLED) {
diskCache.loadMempoolCache();
}
if (config.DATABASE.ENABLED) {
await DB.checkDbConnection();
try {
@@ -115,6 +97,29 @@ class Server {
}
}
this.app
.use((req: Request, res: Response, next: NextFunction) => {
res.setHeader('Access-Control-Allow-Origin', '*');
next();
})
.use(express.urlencoded({ extended: true }))
.use(express.text({ type: ['text/plain', 'application/base64'] }))
;
if (config.DATABASE.ENABLED) {
await priceUpdater.$initializeLatestPriceWithDb();
}
this.server = http.createServer(this.app);
this.wss = new WebSocket.Server({ server: this.server });
this.setUpWebsocketHandling();
await syncAssets.syncAssets$();
if (config.MEMPOOL.ENABLED) {
diskCache.loadMempoolCache();
}
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
statistics.startStatistics();
}
@@ -127,7 +132,7 @@ class Server {
}
}
fiatConversion.startService();
priceUpdater.$run();
this.setUpHttpApiRoutes();
@@ -174,7 +179,7 @@ class Server {
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5;
} catch (e) {
} catch (e: any) {
const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
if (this.currentBackendRetryInterval > 5) {
logger.warn(loggerMsg);
@@ -182,7 +187,9 @@ class Server {
} else {
logger.debug(loggerMsg);
}
logger.debug(JSON.stringify(e));
if (e instanceof AxiosError) {
logger.debug(`AxiosError: ${e?.message}`);
}
setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
this.currentBackendRetryInterval *= 2;
this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
@@ -221,7 +228,7 @@ class Server {
memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
}
fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
}

4
backend/src/repositories/BlocksRepository.ts
View File

@@ -521,7 +521,7 @@ class BlocksRepository {
CAST(AVG(blocks.height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(fees) as INT) as avgFees,
prices.USD
prices.*
FROM blocks
JOIN blocks_prices on blocks_prices.height = blocks.height
JOIN prices on prices.id = blocks_prices.price_id
@@ -550,7 +550,7 @@ class BlocksRepository {
CAST(AVG(blocks.height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(reward) as INT) as avgRewards,
prices.USD
prices.*
FROM blocks
JOIN blocks_prices on blocks_prices.height = blocks.height
JOIN prices on prices.id = blocks_prices.price_id

32
backend/src/repositories/PricesRepository.ts
View File

@@ -1,12 +1,13 @@
import DB from '../database';
import logger from '../logger';
import { Prices } from '../tasks/price-updater';
import { IConversionRates } from '../mempool.interfaces';
import priceUpdater from '../tasks/price-updater';
class PricesRepository {
public async $savePrices(time: number, prices: Prices): Promise<void> {
if (prices.USD === -1) {
// Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
// As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
public async $savePrices(time: number, prices: IConversionRates): Promise<void> {
if (prices.USD === 0) {
// Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues
// As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine
return;
}
@@ -23,22 +24,22 @@ class PricesRepository {
}
public async $getOldestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getLatestPriceId(): Promise<number | null> {
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getPricesTimes(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time`);
return times.map(time => time.time);
}
@@ -46,6 +47,19 @@ class PricesRepository {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
return times;
}
public async $getLatestConversionRates(): Promise<any> {
const [rates]: any[] = await DB.query(`
SELECT USD, EUR, GBP, CAD, CHF, AUD, JPY
FROM prices
ORDER BY time DESC
LIMIT 1`
);
if (!rates || rates.length === 0) {
return priceUpdater.getEmptyPricesObj();
}
return rates[0];
}
}
export default new PricesRepository();

6
backend/src/tasks/price-feeds/bitfinex-api.ts
View File

@@ -13,7 +13,11 @@ class BitfinexApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['last_price'], 10) : -1;
if (response && response['last_price']) {
return parseInt(response['last_price'], 10);
} else {
return -1;
}
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

6
backend/src/tasks/price-feeds/bitflyer-api.ts
View File

@@ -13,7 +13,11 @@ class BitflyerApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['ltp'], 10) : -1;
if (response && response['ltp']) {
return parseInt(response['ltp'], 10);
} else {
return -1;
}
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

6
backend/src/tasks/price-feeds/coinbase-api.ts
View File

@@ -13,7 +13,11 @@ class CoinbaseApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['data']['amount'], 10) : -1;
if (response && response['data'] && response['data']['amount']) {
return parseInt(response['data']['amount'], 10);
} else {
return -1;
}
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

6
backend/src/tasks/price-feeds/gemini-api.ts
View File

@@ -13,7 +13,11 @@ class GeminiApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['last'], 10) : -1;
if (response && response['last']) {
return parseInt(response['last'], 10);
} else {
return -1;
}
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

9
backend/src/tasks/price-feeds/kraken-api.ts
View File

@@ -23,7 +23,14 @@ class KrakenApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
const ticker = this.getTicker(currency);
if (response && response['result'] && response['result'][ticker] &&
response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0
) {
return parseInt(response['result'][ticker]['c'][0], 10);
} else {
return -1;
}
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

65
backend/src/tasks/price-updater.ts
View File

@@ -1,7 +1,8 @@
import * as fs from 'fs';
import path from "path";
import path from 'path';
import config from '../config';
import logger from '../logger';
import { IConversionRates } from '../mempool.interfaces';
import PricesRepository from '../repositories/PricesRepository';
import BitfinexApi from './price-feeds/bitfinex-api';
import BitflyerApi from './price-feeds/bitflyer-api';
@@ -20,17 +21,7 @@ export interface PriceFeed {
}
export interface PriceHistory {
[timestamp: number]: Prices;
}
export interface Prices {
USD: number;
EUR: number;
GBP: number;
CAD: number;
CHF: number;
AUD: number;
JPY: number;
[timestamp: number]: IConversionRates;
}
class PriceUpdater {
@@ -40,7 +31,8 @@ class PriceUpdater {
running = false;
feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: Prices;
latestPrices: IConversionRates;
private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
constructor() {
this.latestPrices = this.getEmptyPricesObj();
@@ -52,18 +44,30 @@ class PriceUpdater {
this.feeds.push(new GeminiApi());
}
public getEmptyPricesObj(): Prices {
public getEmptyPricesObj(): IConversionRates {
return {
USD: -1,
EUR: -1,
GBP: -1,
CAD: -1,
CHF: -1,
AUD: -1,
JPY: -1,
USD: 0,
EUR: 0,
GBP: 0,
CAD: 0,
CHF: 0,
AUD: 0,
JPY: 0,
};
}
public setRatesChangedCallback(fn: (rates: IConversionRates) => void) {
this.ratesChangedCallback = fn;
}
/**
* We execute this function before the websocket initialization since
* the websocket init is not done asyncronously
*/
public async $initializeLatestPriceWithDb(): Promise<void> {
this.latestPrices = await PricesRepository.$getLatestConversionRates();
}
public async $run(): Promise<void> {
if (this.running === true) {
return;
@@ -76,10 +80,9 @@ class PriceUpdater {
}
try {
await this.$updatePrice();
if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
await this.$insertHistoricalPrices();
} else {
await this.$updatePrice();
}
} catch (e) {
logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
@@ -127,7 +130,11 @@ class PriceUpdater {
// Compute average price, non weighted
prices = prices.filter(price => price > 0);
this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
if (prices.length === 0) {
this.latestPrices[currency] = -1;
} else {
this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
}
}
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
@@ -144,6 +151,10 @@ class PriceUpdater {
}
}
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.latestPrices);
}
this.lastRun = new Date().getTime() / 1000;
}
@@ -213,7 +224,7 @@ class PriceUpdater {
// Group them by timestamp and currency, for example
// grouped[123456789]['USD'] = [1, 2, 3, 4];
const grouped: Object = {};
const grouped: any = {};
for (const historicalEntry of historicalPrices) {
for (const time in historicalEntry) {
if (existingPriceTimes.includes(parseInt(time, 10))) {
@@ -229,7 +240,7 @@ class PriceUpdater {
for (const currency of this.currencies) {
const price = historicalEntry[time][currency];
if (price > 0) {
grouped[time][currency].push(parseInt(price, 10));
grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
}
}
}
@@ -238,7 +249,7 @@ class PriceUpdater {
// Average prices and insert everything into the db
let totalInserted = 0;
for (const time in grouped) {
const prices: Prices = this.getEmptyPricesObj();
const prices: IConversionRates = this.getEmptyPricesObj();
for (const currency in grouped[time]) {
if (grouped[time][currency].length === 0) {
continue;

2
backend/src/utils/blocks-utils.ts
View File

@@ -17,7 +17,7 @@ export function prepareBlock(block: any): BlockExtended {
extras: {
coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
feeRange: block.feeRange ?? block.fee_span,
feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span,
reward: block.reward ?? block?.extras?.reward,
totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
avgFee: block?.extras?.avgFee ?? block.avg_fee,