Compare commits

..

2 Commits

Author SHA1 Message Date
Mononaut
cdfee53cf5 Use minfee second node to supply alt rbf policy mempool txs 2023-01-17 19:25:53 -06:00
Mononaut
be27e3df52 check for txs in alternative backend mempool 2023-01-17 19:25:53 -06:00
343 changed files with 31210 additions and 70263 deletions

1
.github/CODEOWNERS vendored
View File

@@ -1 +0,0 @@
backend/src/api/database-migration.ts @wiz @softsimon

View File

@@ -9,7 +9,7 @@ jobs:
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
strategy:
matrix:
node: ["16.16.0", "18.14.1"]
node: ["16.16.0", "18.5.0"]
flavor: ["dev", "prod"]
fail-fast: false
runs-on: "ubuntu-latest"
@@ -55,7 +55,7 @@ jobs:
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
strategy:
matrix:
node: ["16.16.0", "18.14.1"]
node: ["16.15.0", "18.5.0"]
flavor: ["dev", "prod"]
fail-fast: false
runs-on: "ubuntu-latest"

View File

@@ -1,11 +1,8 @@
name: Cypress Tests
on:
push:
branches: [master]
pull_request:
types: [opened, synchronize]
types: [opened, review_requested, synchronize]
jobs:
cypress:
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"

View File

@@ -1,26 +0,0 @@
name: 'Print images digest'
on:
workflow_dispatch:
inputs:
version:
description: 'Image Version'
required: false
default: 'latest'
type: string
jobs:
print-images-sha:
runs-on: 'ubuntu-latest'
name: Print digest for images
steps:
- name: Checkout
uses: actions/checkout@v3
with:
path: digest
- name: Run script
working-directory: digest
run: |
sh ./docker/scripts/get_image_digest.sh $VERSION
env:
VERSION: ${{ github.event.inputs.version }}

View File

@@ -1,5 +1,4 @@
{
"editor.tabSize": 2,
"typescript.preferences.importModuleSpecifier": "relative",
"typescript.tsdk": "./backend/node_modules/typescript/lib"
}

View File

@@ -1,5 +1,5 @@
The Mempool Open Source Project
Copyright (c) 2019-2023 The Mempool Open Source Project Developers
Copyright (c) 2019-2022 The Mempool Open Source Project Developers
This program is free software; you can redistribute it and/or modify it under
the terms of (at your option) either:

View File

@@ -1,13 +1,11 @@
# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
<br>
Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/).
It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
# Installation Methods
Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server.

View File

@@ -160,7 +160,7 @@ npm install -g ts-node nodemon
Then, run the watcher:
```
nodemon src/index.ts --ignore cache/
nodemon src/index.ts --ignore cache/ --ignore pools.json
```
`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -171,84 +171,50 @@ Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe
Run bitcoind on regtest:
```
bitcoind -regtest
bitcoind -regtest -rpcport=8332
```
Create a new wallet, if needed:
```
bitcoin-cli -regtest createwallet test
bitcoin-cli -regtest -rpcport=8332 createwallet test
```
Load wallet (this command may take a while if you have lot of UTXOs):
```
bitcoin-cli -regtest loadwallet test
bitcoin-cli -regtest -rpcport=8332 loadwallet test
```
Get a new address:
```
address=$(bitcoin-cli -regtest getnewaddress)
address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
```
Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
```
bitcoin-cli -regtest generatetoaddress 101 $address
bitcoin-cli -regtest -rpcport=8332 generatetoaddress 101 $address
```
Send 0.1 BTC at 5 sat/vB to another address:
```
bitcoin-cli -named -regtest sendtoaddress address=$(bitcoin-cli -regtest getnewaddress) amount=0.1 fee_rate=5
./src/bitcoin-cli -named -regtest -rpcport=8332 sendtoaddress address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress) amount=0.1 fee_rate=5
```
See more example of `sendtoaddress`:
```
bitcoin-cli sendtoaddress # will print the help
./src/bitcoin-cli sendtoaddress # will print the help
```
Mini script to generate random network activity (random TX count with random tx fee-rate). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
Mini script to generate transactions with random TX fee-rate (between 1 to 100 sat/vB). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
```
#!/bin/bash
address=$(bitcoin-cli -regtest getnewaddress)
bitcoin-cli -regtest generatetoaddress 101 $address
address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
for i in {1..1000000}
do
for y in $(seq 1 "$(jot -r 1 1 1000)")
do
bitcoin-cli -regtest -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1 1 100)
done
bitcoin-cli -regtest generatetoaddress 1 $address
sleep 5
./src/bitcoin-cli -regtest -rpcport=8332 -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1 1 100)
done
```
Generate block at regular interval (every 10 seconds in this example):
```
watch -n 10 "bitcoin-cli -regtest generatetoaddress 1 $address"
watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
```
### Mining pools update
By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`).
To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
### Re-index tables
You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
Usage:
```
npm run start --reindex=blocks,hashrates
```
Example output:
```
Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
```
Reference: https://github.com/mempool/mempool/pull/1269

View File

@@ -15,6 +15,7 @@
"MEMPOOL_BLOCKS_AMOUNT": 8,
"INDEXING_BLOCKS_AMOUNT": 11000,
"BLOCKS_SUMMARIES_INDEXING": false,
"PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [],
"EXTERNAL_MAX_RETRY": 1,
@@ -22,20 +23,18 @@
"USER_AGENT": "mempool",
"STDOUT_LOG_MIN_PRIORITY": "debug",
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"AUDIT": false,
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6
"RBF_DUAL_NODE": false
},
"CORE_RPC": {
"HOST": "127.0.0.1",
"PORT": 8332,
"USERNAME": "mempool",
"PASSWORD": "mempool",
"TIMEOUT": 60000
"PASSWORD": "mempool"
},
"ELECTRUM": {
"HOST": "127.0.0.1",
@@ -49,8 +48,7 @@
"HOST": "127.0.0.1",
"PORT": 8332,
"USERNAME": "mempool",
"PASSWORD": "mempool",
"TIMEOUT": 60000
"PASSWORD": "mempool"
},
"DATABASE": {
"ENABLED": true,
@@ -94,8 +92,7 @@
"LND": {
"TLS_CERT_PATH": "tls.cert",
"MACAROON_PATH": "readonly.macaroon",
"REST_API_URL": "https://localhost:8080",
"TIMEOUT": 10000
"REST_API_URL": "https://localhost:8080"
},
"CLIGHTNING": {
"SOCKET": "lightning-rpc"

View File

@@ -27,7 +27,7 @@
"package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
"package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
"start": "node --max-old-space-size=2048 dist/index.js",
"start-production": "node --max-old-space-size=16384 dist/index.js",
"start-production": "node --max-old-space-size=4096 dist/index.js",
"test": "./node_modules/.bin/jest --coverage",
"lint": "./node_modules/.bin/eslint . --ext .ts",
"lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",

View File

@@ -3,11 +3,12 @@
"ENABLED": true,
"NETWORK": "__MEMPOOL_NETWORK__",
"BACKEND": "__MEMPOOL_BACKEND__",
"ENABLED": true,
"BLOCKS_SUMMARIES_INDEXING": true,
"HTTP_PORT": 1,
"SPAWN_CLUSTER_PROCS": 2,
"API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
"AUTOMATIC_BLOCK_REINDEXING": false,
"AUTOMATIC_BLOCK_REINDEXING": true,
"POLL_RATE_MS": 3,
"CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
"CLEAR_PROTECTION_MINUTES": 4,
@@ -15,6 +16,7 @@
"BLOCK_WEIGHT_UNITS": 6,
"INITIAL_BLOCKS_AMOUNT": 7,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"PRICE_FEED_UPDATE_INTERVAL": 9,
"USE_SECOND_NODE_FOR_MINFEE": 10,
"EXTERNAL_ASSETS": 11,
"EXTERNAL_MAX_RETRY": 12,
@@ -24,19 +26,16 @@
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"AUDIT": "__MEMPOOL_AUDIT__",
"ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
"MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
"DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
"ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__CPFP_INDEXING__",
"RBF_DUAL_NODE": "__RBF_DUAL_NODE__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",
"PORT": 15,
"USERNAME": "__CORE_RPC_USERNAME__",
"PASSWORD": "__CORE_RPC_PASSWORD__",
"TIMEOUT": "__CORE_RPC_TIMEOUT__"
"PASSWORD": "__CORE_RPC_PASSWORD__"
},
"ELECTRUM": {
"HOST": "__ELECTRUM_HOST__",
@@ -50,8 +49,7 @@
"HOST": "__SECOND_CORE_RPC_HOST__",
"PORT": 17,
"USERNAME": "__SECOND_CORE_RPC_USERNAME__",
"PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
"TIMEOUT": "__SECOND_CORE_RPC_TIMEOUT__"
"PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
},
"DATABASE": {
"ENABLED": false,
@@ -110,8 +108,7 @@
"LND": {
"TLS_CERT_PATH": "",
"MACAROON_PATH": "",
"REST_API_URL": "https://localhost:8080",
"TIMEOUT": 10000
"REST_API_URL": "https://localhost:8080"
},
"CLIGHTNING": {
"SOCKET": "__CLIGHTNING_SOCKET__"

View File

@@ -23,11 +23,9 @@ describe('Mempool Difficulty Adjustment', () => {
remainingBlocks: 1834,
remainingTime: 977591692,
previousRetarget: 0.6280047707459726,
previousTime: 1660820820,
nextRetargetHeight: 751968,
timeAvg: 533038,
timeOffset: 0,
expectedBlocks: 161.68833333333333,
},
],
[ // Vector 2 (testnet)
@@ -45,13 +43,11 @@ describe('Mempool Difficulty Adjustment', () => {
estimatedRetargetDate: 1661895424692,
remainingBlocks: 1834,
remainingTime: 977591692,
previousTime: 1660820820,
previousRetarget: 0.6280047707459726,
nextRetargetHeight: 751968,
timeAvg: 533038,
timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
// If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
expectedBlocks: 161.68833333333333,
},
],
] as [[number, number, number, number, string, number], DifficultyAdjustment][];

View File

@@ -29,6 +29,7 @@ describe('Mempool Backend Config', () => {
INITIAL_BLOCKS_AMOUNT: 8,
MEMPOOL_BLOCKS_AMOUNT: 8,
INDEXING_BLOCKS_AMOUNT: 11000,
PRICE_FEED_UPDATE_INTERVAL: 600,
USE_SECOND_NODE_FOR_MINFEE: false,
EXTERNAL_ASSETS: [],
EXTERNAL_MAX_RETRY: 1,
@@ -36,13 +37,11 @@ describe('Mempool Backend Config', () => {
USER_AGENT: 'mempool',
STDOUT_LOG_MIN_PRIORITY: 'debug',
POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
AUDIT: false,
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
CPFP_INDEXING: false,
MAX_BLOCKS_BULK_QUERY: 0,
DISK_CACHE_BLOCK_INTERVAL: 6,
RBF_DUAL_NODE: false,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
@@ -53,16 +52,14 @@ describe('Mempool Backend Config', () => {
HOST: '127.0.0.1',
PORT: 8332,
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 60000
PASSWORD: 'mempool'
});
expect(config.SECOND_CORE_RPC).toStrictEqual({
HOST: '127.0.0.1',
PORT: 8332,
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 60000
PASSWORD: 'mempool'
});
expect(config.DATABASE).toStrictEqual({
@@ -109,13 +106,6 @@ describe('Mempool Backend Config', () => {
BISQ_URL: 'https://bisq.markets/api',
BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
});
expect(config.MAXMIND).toStrictEqual({
ENABLED: false,
GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
});
});
});

View File

@@ -0,0 +1,175 @@
import config from '../config';
import { TransactionExtended } from '../mempool.interfaces';
import logger from '../logger';
import { Common } from './common';
import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
import BitcoinApi from './bitcoin/bitcoin-api';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
import { Mempool } from './mempool';
class AltMempool extends Mempool {
private bitcoinSecondApi: BitcoinApi;
constructor() {
super();
this.bitcoinSecondApi = new BitcoinApi(bitcoinSecondClient, this, bitcoinApi);
}
protected init(): void {
// override
}
public setOutOfSync(): void {
this.inSync = false;
}
public setMempool(mempoolData: { [txId: string]: TransactionExtended }): void {
this.mempoolCache = mempoolData;
}
public getFirstSeenForTransactions(txIds: string[]): number[] {
const txTimes: number[] = [];
txIds.forEach((txId: string) => {
const tx = this.mempoolCache[txId];
if (tx && tx.firstSeen) {
txTimes.push(tx.firstSeen);
} else {
txTimes.push(0);
}
});
return txTimes;
}
public async $updateMempool(): Promise<void> {
logger.debug(`Updating alternative mempool...`);
const start = new Date().getTime();
const currentMempoolSize = Object.keys(this.mempoolCache).length;
const transactions = await this.bitcoinSecondApi.$getRawMempool();
const diff = transactions.length - currentMempoolSize;
this.mempoolCacheDelta = Math.abs(diff);
const loadingMempool = this.mempoolCacheDelta > 100;
for (const txid of transactions) {
if (!this.mempoolCache[txid]) {
try {
const transaction = await this.$fetchTransaction(txid);
this.mempoolCache[txid] = transaction;
if (loadingMempool && Object.keys(this.mempoolCache).length % 50 === 0) {
logger.info(`loaded ${Object.keys(this.mempoolCache).length}/${transactions.length} alternative mempool transactions`);
}
} catch (e) {
logger.debug(`Error finding transaction '${txid}' in the alternative mempool: ` + (e instanceof Error ? e.message : e));
}
}
if ((new Date().getTime()) - start > Mempool.WEBSOCKET_REFRESH_RATE_MS) {
break;
}
}
if (loadingMempool) {
logger.info(`loaded ${Object.keys(this.mempoolCache).length}/${transactions.length} alternative mempool transactions`);
}
// Prevent mempool from clear on bitcoind restart by delaying the deletion
if (this.mempoolProtection === 0
&& currentMempoolSize > 20000
&& transactions.length / currentMempoolSize <= 0.80
) {
this.mempoolProtection = 1;
setTimeout(() => {
this.mempoolProtection = 2;
}, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
}
const deletedTransactions: string[] = [];
if (this.mempoolProtection !== 1) {
this.mempoolProtection = 0;
// Index object for faster search
const transactionsObject = {};
transactions.forEach((txId) => transactionsObject[txId] = true);
// Flag transactions for lazy deletion
for (const tx in this.mempoolCache) {
if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
deletedTransactions.push(this.mempoolCache[tx].txid);
}
}
for (const txid of deletedTransactions) {
delete this.mempoolCache[txid];
}
}
this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
const end = new Date().getTime();
const time = end - start;
logger.debug(`Alt mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
}
public getTransaction(txid: string): TransactionExtended {
return this.mempoolCache[txid] || null;
}
protected async $fetchTransaction(txid: string): Promise<TransactionExtended> {
const rawTx = await this.bitcoinSecondApi.$getRawTransaction(txid, false, true, false);
return this.extendTransaction(rawTx);
}
protected extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
// @ts-ignore
if (transaction.vsize) {
// @ts-ignore
return transaction;
}
const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
(transaction.fee || 0) / (transaction.weight / 4));
const transactionExtended: TransactionExtended = Object.assign({
vsize: Math.round(transaction.weight / 4),
feePerVsize: feePerVbytes,
effectiveFeePerVsize: feePerVbytes,
}, transaction);
if (!transaction.status.confirmed) {
transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));
}
return transactionExtended;
}
public async $updateMemPoolInfo(): Promise<void> {
this.mempoolInfo = await this.$getMempoolInfo();
}
public getMempoolInfo(): IBitcoinApi.MempoolInfo {
return this.mempoolInfo;
}
public getTxPerSecond(): number {
return this.txPerSecond;
}
public getVBytesPerSecond(): number {
return this.vBytesPerSecond;
}
public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }): void {
for (const rbfTransaction in rbfTransactions) {
if (this.mempoolCache[rbfTransaction]) {
// Erase the replaced transactions from the local mempool
delete this.mempoolCache[rbfTransaction];
}
}
}
protected updateTxPerSecond(): void {}
protected deleteExpiredTransactions(): void {}
protected $getMempoolInfo(): any {
return bitcoinSecondClient.getMempoolInfo();
}
}
export default new AltMempool();

View File

@@ -5,9 +5,9 @@ const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first
class Audit {
auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
: { censored: string[], added: string[], fresh: string[], score: number, similarity: number } {
: { censored: string[], added: string[], fresh: string[], score: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], fresh: [], score: 0, similarity: 1 };
return { censored: [], added: [], fresh: [], score: 0 };
}
const matches: string[] = []; // present in both mined block and template
@@ -16,8 +16,6 @@ class Audit {
const isCensored = {}; // missing, without excuse
const isDisplaced = {};
let displacedWeight = 0;
let matchedWeight = 0;
let projectedWeight = 0;
const inBlock = {};
const inTemplate = {};
@@ -40,16 +38,11 @@ class Audit {
isCensored[txid] = true;
}
displacedWeight += mempool[txid].weight;
} else {
matchedWeight += mempool[txid].weight;
}
projectedWeight += mempool[txid].weight;
inTemplate[txid] = true;
}
displacedWeight += (4000 - transactions[0].weight);
projectedWeight += transactions[0].weight;
matchedWeight += transactions[0].weight;
// we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
// these displaced transactions should occupy the first N weight units of the next projected block
@@ -126,16 +119,13 @@ class Audit {
}
const numCensored = Object.keys(isCensored).length;
const numMatches = matches.length - 1; // adjust for coinbase tx
const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
return {
censored: Object.keys(isCensored),
added,
fresh,
score,
similarity,
score
};
}
}

View File

@@ -4,19 +4,20 @@ import EsploraApi from './esplora-api';
import BitcoinApi from './bitcoin-api';
import ElectrumApi from './electrum-api';
import bitcoinClient from './bitcoin-client';
import mempool from '../mempool';
function bitcoinApiFactory(): AbstractBitcoinApi {
switch (config.MEMPOOL.BACKEND) {
case 'esplora':
return new EsploraApi();
case 'electrum':
return new ElectrumApi(bitcoinClient);
return new ElectrumApi(bitcoinClient, mempool);
case 'none':
default:
return new BitcoinApi(bitcoinClient);
return new BitcoinApi(bitcoinClient, mempool);
}
}
export const bitcoinCoreApi = new BitcoinApi(bitcoinClient);
export const bitcoinCoreApi = new BitcoinApi(bitcoinClient, mempool);
export default bitcoinApiFactory();

View File

@@ -172,35 +172,4 @@ export namespace IBitcoinApi {
}
}
export interface BlockStats {
"avgfee": number;
"avgfeerate": number;
"avgtxsize": number;
"blockhash": string;
"feerate_percentiles": [number, number, number, number, number];
"height": number;
"ins": number;
"maxfee": number;
"maxfeerate": number;
"maxtxsize": number;
"medianfee": number;
"mediantime": number;
"mediantxsize": number;
"minfee": number;
"minfeerate": number;
"mintxsize": number;
"outs": number;
"subsidy": number;
"swtotal_size": number;
"swtotal_weight": number;
"swtxs": number;
"time": number;
"total_out": number;
"total_size": number;
"total_weight": number;
"totalfee": number;
"txs": number;
"utxo_increase": number;
"utxo_size_inc": number;
}
}

View File

@@ -3,15 +3,18 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IBitcoinApi } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
import blocks from '../blocks';
import mempool from '../mempool';
import { TransactionExtended } from '../../mempool.interfaces';
import { MempoolBlock, TransactionExtended } from '../../mempool.interfaces';
class BitcoinApi implements AbstractBitcoinApi {
private mempool: any = null;
private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
protected bitcoindClient: any;
protected backupBitcoinApi: BitcoinApi;
constructor(bitcoinClient: any) {
constructor(bitcoinClient: any, mempool: any, backupBitcoinApi: any = null) {
this.bitcoindClient = bitcoinClient;
this.mempool = mempool;
this.backupBitcoinApi = backupBitcoinApi;
}
static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
@@ -28,14 +31,13 @@ class BitcoinApi implements AbstractBitcoinApi {
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
mediantime: block.mediantime,
};
}
$getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
// If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
const txInMempool = mempool.getMempool()[txId];
const txInMempool = this.mempool.getMempool()[txId];
if (txInMempool && addPrevout) {
return this.$addPrevouts(txInMempool);
}
@@ -119,7 +121,7 @@ class BitcoinApi implements AbstractBitcoinApi {
$getAddressPrefix(prefix: string): string[] {
const found: { [address: string]: string } = {};
const mp = mempool.getMempool();
const mp = this.mempool.getMempool();
for (const tx in mp) {
for (const vout of mp[tx].vout) {
if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
@@ -261,7 +263,7 @@ class BitcoinApi implements AbstractBitcoinApi {
return transaction;
}
let mempoolEntry: IBitcoinApi.MempoolEntry;
if (!mempool.isInSync() && !this.rawMempoolCache) {
if (!this.mempool.isInSync() && !this.rawMempoolCache) {
this.rawMempoolCache = await this.$getRawMempoolVerbose();
}
if (this.rawMempoolCache && this.rawMempoolCache[transaction.txid]) {
@@ -317,10 +319,23 @@ class BitcoinApi implements AbstractBitcoinApi {
transaction.vin[i].lazy = true;
continue;
}
const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
this.addInnerScriptsToVin(transaction.vin[i]);
totalIn += innerTx.vout[transaction.vin[i].vout].value;
let innerTx;
try {
innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
} catch (e) {
if (this.backupBitcoinApi) {
// input tx is confirmed, but the preferred client has txindex=0, so fetch from the backup client instead.
const backupTx = await this.backupBitcoinApi.$getRawTransaction(transaction.vin[i].txid);
innerTx = JSON.parse(JSON.stringify(backupTx));
} else {
throw e;
}
}
if (innerTx) {
transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
this.addInnerScriptsToVin(transaction.vin[i]);
totalIn += innerTx.vout[transaction.vin[i].vout].value;
}
}
if (lazyPrevouts && transaction.vin.length > 12) {
transaction.fee = -1;

View File

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
port: config.CORE_RPC.PORT,
user: config.CORE_RPC.USERNAME,
pass: config.CORE_RPC.PASSWORD,
timeout: config.CORE_RPC.TIMEOUT,
timeout: 60000,
};
export default new bitcoin.Client(nodeRpcCredentials);

View File

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
port: config.SECOND_CORE_RPC.PORT,
user: config.SECOND_CORE_RPC.USERNAME,
pass: config.SECOND_CORE_RPC.PASSWORD,
timeout: config.SECOND_CORE_RPC.TIMEOUT,
timeout: 60000,
};
export default new bitcoin.Client(nodeRpcCredentials);

View File

@@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler';
import mempool from '../mempool';
import feeApi from '../fee-api';
import mempoolBlocks from '../mempool-blocks';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
import bitcoinApi from './bitcoin-api-factory';
import { Common } from '../common';
import backendInfo from '../backend-info';
import transactionUtils from '../transaction-utils';
@@ -19,6 +19,7 @@ import bitcoinClient from './bitcoin-client';
import difficultyAdjustment from '../difficulty-adjustment';
import transactionRepository from '../../repositories/TransactionRepository';
import rbfCache from '../rbf-cache';
import altMempool from '../alt-mempool';
class BitcoinRoutes {
public initRoutes(app: Application) {
@@ -32,6 +33,7 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
.get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/alt/:txId', this.getAltTransaction)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
.post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
@@ -95,8 +97,6 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
.post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
;
if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -217,20 +217,13 @@ class BitcoinRoutes {
res.json(cpfpInfo);
return;
} else {
let cpfpInfo;
if (config.DATABASE.ENABLED) {
cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
}
const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
if (cpfpInfo) {
res.json(cpfpInfo);
return;
} else {
res.json({
ancestors: []
});
return;
}
}
res.status(404).send(`Transaction has no CPFP info available.`);
}
private getBackendInfo(req: Request, res: Response) {
@@ -250,6 +243,23 @@ class BitcoinRoutes {
}
}
private async getAltTransaction(req: Request, res: Response) {
try {
const transaction = altMempool.getTransaction(req.params.txId);
if (transaction) {
res.json(transaction);
} else {
res.status(404).send('No such transaction in the alternate mempool');
}
} catch (e) {
let statusCode = 500;
if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
statusCode = 404;
}
res.status(statusCode).send(e instanceof Error ? e.message : e);
}
}
private async getRawTransaction(req: Request, res: Response) {
try {
const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true);
@@ -411,41 +421,6 @@ class BitcoinRoutes {
}
}
private async getBlocksByBulk(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
return res.status(404).send(`This API is only available for Bitcoin networks`);
}
if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
}
if (!Common.indexingEnabled()) {
return res.status(404).send(`Indexing is required for this API`);
}
const from = parseInt(req.params.from, 10);
if (!req.params.from || from < 0) {
return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
}
const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
if (to < 0) {
return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
}
if (from > to) {
return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
}
if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
}
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await blocks.$getBlocksBetweenHeight(from, to));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getLegacyBlocks(req: Request, res: Response) {
try {
const returnBlocks: IEsploraApi.Block[] = [];
@@ -468,7 +443,7 @@ class BitcoinRoutes {
returnBlocks.push(localBlock);
nextHash = localBlock.previousblockhash;
} else {
const block = await bitcoinCoreApi.$getBlock(nextHash);
const block = await bitcoinApi.$getBlock(nextHash);
returnBlocks.push(block);
nextHash = block.previousblockhash;
}
@@ -651,7 +626,7 @@ class BitcoinRoutes {
if (result) {
res.json(result);
} else {
res.status(204).send();
res.status(404).send('not found');
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);

View File

@@ -12,8 +12,8 @@ import memoryCache from '../memory-cache';
class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
private electrumClient: any;
constructor(bitcoinClient: any) {
super(bitcoinClient);
constructor(bitcoinClient: any, mempool: any) {
super(bitcoinClient, mempool);
const electrumConfig = { client: 'mempool-v2', version: '1.4' };
const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };

View File

@@ -88,7 +88,6 @@ export namespace IEsploraApi {
size: number;
weight: number;
previousblockhash: string;
mediantime: number;
}
export interface Address {

View File

@@ -1,13 +1,8 @@
import config from '../../config';
import axios, { AxiosRequestConfig } from 'axios';
import http from 'http';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IEsploraApi } from './esplora-api.interface';
const axiosConnection = axios.create({
httpAgent: new http.Agent({ keepAlive: true })
});
class ElectrsApi implements AbstractBitcoinApi {
axiosConfig: AxiosRequestConfig = {
timeout: 10000,
@@ -16,52 +11,52 @@ class ElectrsApi implements AbstractBitcoinApi {
constructor() { }
$getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
.then((response) => response.data);
}
$getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
.then((response) => response.data);
}
$getTransactionHex(txId: string): Promise<string> {
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
return axios.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
.then((response) => response.data);
}
$getBlockHeightTip(): Promise<number> {
return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
.then((response) => response.data);
}
$getBlockHashTip(): Promise<string> {
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
.then((response) => response.data);
}
$getTxIdsForBlock(hash: string): Promise<string[]> {
return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
.then((response) => response.data);
}
$getBlockHash(height: number): Promise<string> {
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
.then((response) => response.data);
}
$getBlockHeader(hash: string): Promise<string> {
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
.then((response) => response.data);
}
$getBlock(hash: string): Promise<IEsploraApi.Block> {
return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
.then((response) => response.data);
}
$getRawBlock(hash: string): Promise<Buffer> {
return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
.then((response) => { return Buffer.from(response.data); });
}
@@ -82,12 +77,12 @@ class ElectrsApi implements AbstractBitcoinApi {
}
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
.then((response) => response.data);
}
$getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
.then((response) => response.data);
}

View File

@@ -1,8 +1,8 @@
import config from '../config';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
@@ -13,9 +13,11 @@ import poolsRepository from '../repositories/PoolsRepository';
import blocksRepository from '../repositories/BlocksRepository';
import loadingIndicators from './loading-indicators';
import BitcoinApi from './bitcoin/bitcoin-api';
import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
import fiatConversion from './fiat-conversion';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
@@ -24,7 +26,6 @@ import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
import chainTips from './chain-tips';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -142,11 +143,8 @@ class Blocks {
* @param block
* @returns BlockSummary
*/
public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
if (Common.isLiquid()) {
block = this.convertLiquidFees(block);
}
const stripped = block.tx.map((tx: IBitcoinApi.VerboseTransaction) => {
private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
const stripped = block.tx.map((tx) => {
return {
txid: tx.txid,
vsize: tx.weight / 4,
@@ -161,13 +159,6 @@ class Blocks {
};
}
private convertLiquidFees(block: IBitcoinApi.VerboseBlock): IBitcoinApi.VerboseBlock {
block.tx.forEach(tx => {
tx.fee = Object.values(tx.fee || {}).reduce((total, output) => total + output, 0);
});
return block;
}
/**
* Return a block with additional data (reward, coinbase, fees...)
* @param block
@@ -175,81 +166,33 @@ class Blocks {
* @returns BlockExtended
*/
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const blk: Partial<BlockExtended> = Object.assign({}, block);
const extras: Partial<BlockExtension> = {};
extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
blockExtended.extras.usd = fiatConversion.getConversionRates().USD;
if (block.height === 0) {
extras.medianFee = 0; // 50th percentiles
extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
extras.totalFees = 0;
extras.avgFee = 0;
extras.avgFeeRate = 0;
extras.utxoSetChange = 0;
extras.avgTxSize = 0;
extras.totalInputs = 0;
extras.totalOutputs = 1;
extras.totalOutputAmt = 0;
extras.segwitTotalTxs = 0;
extras.segwitTotalSize = 0;
extras.segwitTotalWeight = 0;
blockExtended.extras.medianFee = 0; // 50th percentiles
blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
blockExtended.extras.totalFees = 0;
blockExtended.extras.avgFee = 0;
blockExtended.extras.avgFeeRate = 0;
} else {
const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
extras.totalFees = stats.totalfee;
extras.avgFee = stats.avgfee;
extras.avgFeeRate = stats.avgfeerate;
extras.utxoSetChange = stats.utxo_increase;
extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
extras.totalInputs = stats.ins;
extras.totalOutputs = stats.outs;
extras.totalOutputAmt = stats.total_out;
extras.segwitTotalTxs = stats.swtxs;
extras.segwitTotalSize = stats.swtotal_size;
extras.segwitTotalWeight = stats.swtotal_weight;
}
if (Common.blocksSummariesIndexingEnabled()) {
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
if (extras.feePercentiles !== null) {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
extras.virtualSize = block.weight / 4.0;
if (coinbaseTx?.vout.length > 0) {
extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
} else {
extras.coinbaseAddress = null;
extras.coinbaseSignature = null;
extras.coinbaseSignatureAscii = null;
}
const header = await bitcoinClient.getBlockHeader(block.id, false);
extras.header = header;
const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
extras.utxoSetSize = txoutset.txouts,
extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
} else {
extras.utxoSetSize = null;
extras.totalInputAmt = null;
const stats = await bitcoinClient.getBlockStats(block.id, [
'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
]);
blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
blockExtended.extras.totalFees = stats.totalfee;
blockExtended.extras.avgFee = stats.avgfee;
blockExtended.extras.avgFeeRate = stats.avgfeerate;
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
let pool: PoolTag;
if (coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(coinbaseTx);
if (blockExtended.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
} else {
if (config.DATABASE.ENABLED === true) {
pool = await poolsRepository.$getUnknownPool();
@@ -259,27 +202,23 @@ class Blocks {
}
if (!pool) { // We should never have this situation in practise
logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
`Check your "pools" table entries`);
} else {
extras.pool = {
id: pool.uniqueId,
blockExtended.extras.pool = {
id: pool.id,
name: pool.name,
slug: pool.slug,
};
}
extras.matchRate = null;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
extras.matchRate = auditScore.matchRate;
}
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blockExtended.extras.matchRate = auditScore.matchRate;
}
}
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
return blockExtended;
}
/**
@@ -305,18 +244,15 @@ class Blocks {
} else {
pools = poolsParser.miningPools;
}
for (let i = 0; i < pools.length; ++i) {
if (address !== undefined) {
const addresses: string[] = typeof pools[i].addresses === 'string' ?
JSON.parse(pools[i].addresses) : pools[i].addresses;
const addresses: string[] = JSON.parse(pools[i].addresses);
if (addresses.indexOf(address) !== -1) {
return pools[i];
}
}
const regexes: string[] = typeof pools[i].regexes === 'string' ?
JSON.parse(pools[i].regexes) : pools[i].regexes;
const regexes: string[] = JSON.parse(pools[i].regexes);
for (let y = 0; y < regexes.length; ++y) {
const regex = new RegExp(regexes[y], 'i');
const match = asciiScriptSig.match(regex);
@@ -494,7 +430,7 @@ class Blocks {
loadingIndicators.setProgress('block-indexing', progress, false);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -542,13 +478,13 @@ class Blocks {
if (blockchainInfo.blocks === blockchainInfo.headers) {
const heightDiff = blockHeightTip % 2016;
const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty;
if (blockHeightTip >= 2016) {
const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
logger.debug(`Initial difficulty adjustment data set.`);
}
@@ -563,7 +499,6 @@ class Blocks {
} else {
this.currentBlockHeight++;
logger.debug(`New block found (#${this.currentBlockHeight})!`);
await chainTips.updateOrphanedBlocks();
}
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -580,18 +515,18 @@ class Blocks {
if (Common.indexingEnabled()) {
if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
// We assume there won't be a reorg with more than 10 block depth
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await HashratesRepository.$deleteLastEntries();
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock.height - i);
const newBlock = await this.$indexBlock(lastBlock['height'] - i);
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
if (config.MEMPOOL.CPFP_INDEXING) {
await this.$indexCPFP(newBlock.id, lastBlock.height - i);
await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
}
}
await mining.$indexDifficultyAdjustments();
@@ -651,7 +586,7 @@ class Blocks {
if (this.newBlockCallbacks.length) {
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
}
if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
if (!memPool.hasPriority()) {
diskCache.$saveCacheToDisk();
}
@@ -664,27 +599,23 @@ class Blocks {
* Index a block if it's missing from the database. Returns the block after indexing
*/
public async $indexBlock(height: number): Promise<BlockExtended> {
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock !== null) {
return dbBlock;
}
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock != null) {
return prepareBlock(dbBlock);
}
const blockHash = await bitcoinApi.$getBlockHash(height);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
if (Common.indexingEnabled()) {
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
return blockExtended;
return prepareBlock(blockExtended);
}
/**
* Get one block by its hash
* Index a block by hash if it's missing from the database. Returns the block after indexing
*/
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
// Check the memory cache
@@ -693,14 +624,31 @@ class Blocks {
return blockByHash;
}
// Not Bitcoin network, return the block as it from the bitcoin backend
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return await bitcoinCoreApi.$getBlock(hash);
// Block has already been indexed
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHash(hash);
if (dbBlock != null) {
return prepareBlock(dbBlock);
}
}
// Not Bitcoin network, return the block as it
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return await bitcoinApi.$getBlock(hash);
}
let block = await bitcoinClient.getBlock(hash);
block = prepareBlock(block);
// Bitcoin network, add our custom data on top
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
return await this.$indexBlock(block.height);
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
if (Common.indexingEnabled()) {
delete(blockExtended['coinbaseTx']);
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return blockExtended;
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@@ -734,19 +682,8 @@ class Blocks {
return summary.transactions;
}
/**
* Get 15 blocks
*
* Internally this function uses two methods to get the blocks, and
* the method is automatically selected:
* - Using previous block hash links
* - Using block height
*
* @param fromHeight
* @param limit
* @returns
*/
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
if (currentHeight > this.currentBlockHeight) {
limit -= currentHeight - this.currentBlockHeight;
@@ -758,15 +695,27 @@ class Blocks {
return returnBlocks;
}
// Check if block height exist in local cache to skip the hash lookup
const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
let startFromHash: string | null = null;
if (blockByHeight) {
startFromHash = blockByHeight.id;
} else if (!Common.indexingEnabled()) {
startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
}
let nextHash = startFromHash;
for (let i = 0; i < limit && currentHeight >= 0; i++) {
let block = this.getBlocks().find((b) => b.height === currentHeight);
if (block) {
// Using the memory cache (find by height)
returnBlocks.push(block);
} else {
// Using indexing (find by height, index on the fly, save in database)
} else if (Common.indexingEnabled()) {
block = await this.$indexBlock(currentHeight);
returnBlocks.push(block);
} else if (nextHash != null) {
block = prepareBlock(await bitcoinClient.getBlock(nextHash));
nextHash = block.previousblockhash;
returnBlocks.push(block);
}
currentHeight--;
}
@@ -774,114 +723,6 @@ class Blocks {
return returnBlocks;
}
/**
* Used for bulk block data query
*
* @param fromHeight
* @param toHeight
*/
public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
if (!Common.indexingEnabled()) {
return [];
}
const blocks: any[] = [];
while (fromHeight <= toHeight) {
let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
await this.$indexBlock(fromHeight);
block = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
continue;
}
}
// Cleanup fields before sending the response
const cleanBlock: any = {
height: block.height ?? null,
hash: block.id ?? null,
timestamp: block.timestamp ?? null,
median_timestamp: block.mediantime ?? null,
previous_block_hash: block.previousblockhash ?? null,
difficulty: block.difficulty ?? null,
header: block.extras.header ?? null,
version: block.version ?? null,
bits: block.bits ?? null,
nonce: block.nonce ?? null,
size: block.size ?? null,
weight: block.weight ?? null,
tx_count: block.tx_count ?? null,
merkle_root: block.merkle_root ?? null,
reward: block.extras.reward ?? null,
total_fee_amt: block.extras.totalFees ?? null,
avg_fee_amt: block.extras.avgFee ?? null,
median_fee_amt: block.extras.medianFeeAmt ?? null,
fee_amt_percentiles: block.extras.feePercentiles ?? null,
avg_fee_rate: block.extras.avgFeeRate ?? null,
median_fee_rate: block.extras.medianFee ?? null,
fee_rate_percentiles: block.extras.feeRange ?? null,
total_inputs: block.extras.totalInputs ?? null,
total_input_amt: block.extras.totalInputAmt ?? null,
total_outputs: block.extras.totalOutputs ?? null,
total_output_amt: block.extras.totalOutputAmt ?? null,
segwit_total_txs: block.extras.segwitTotalTxs ?? null,
segwit_total_size: block.extras.segwitTotalSize ?? null,
segwit_total_weight: block.extras.segwitTotalWeight ?? null,
avg_tx_size: block.extras.avgTxSize ?? null,
utxoset_change: block.extras.utxoSetChange ?? null,
utxoset_size: block.extras.utxoSetSize ?? null,
coinbase_raw: block.extras.coinbaseRaw ?? null,
coinbase_address: block.extras.coinbaseAddress ?? null,
coinbase_signature: block.extras.coinbaseSignature ?? null,
coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
pool_slug: block.extras.pool.slug ?? null,
pool_id: block.extras.pool.id ?? null,
};
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
if (cleanBlock.fee_amt_percentiles === null) {
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
const summary = this.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
}
if (cleanBlock.fee_amt_percentiles !== null) {
cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
}
}
cleanBlock.fee_amt_percentiles = {
'min': cleanBlock.fee_amt_percentiles[0],
'perc_10': cleanBlock.fee_amt_percentiles[1],
'perc_25': cleanBlock.fee_amt_percentiles[2],
'perc_50': cleanBlock.fee_amt_percentiles[3],
'perc_75': cleanBlock.fee_amt_percentiles[4],
'perc_90': cleanBlock.fee_amt_percentiles[5],
'max': cleanBlock.fee_amt_percentiles[6],
};
cleanBlock.fee_rate_percentiles = {
'min': cleanBlock.fee_rate_percentiles[0],
'perc_10': cleanBlock.fee_rate_percentiles[1],
'perc_25': cleanBlock.fee_rate_percentiles[2],
'perc_50': cleanBlock.fee_rate_percentiles[3],
'perc_75': cleanBlock.fee_rate_percentiles[4],
'perc_90': cleanBlock.fee_rate_percentiles[5],
'max': cleanBlock.fee_rate_percentiles[6],
};
// Re-org can happen after indexing so we need to always get the
// latest state from core
cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
blocks.push(cleanBlock);
fromHeight++;
}
return blocks;
}
public async $getBlockAuditSummary(hash: string): Promise<any> {
let summary;
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {

View File

@@ -1,61 +0,0 @@
import logger from '../logger';
import bitcoinClient from './bitcoin/bitcoin-client';
export interface ChainTip {
height: number;
hash: string;
branchlen: number;
status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
};
export interface OrphanedBlock {
height: number;
hash: string;
status: 'valid-fork' | 'valid-headers' | 'headers-only';
}
class ChainTips {
private chainTips: ChainTip[] = [];
private orphanedBlocks: OrphanedBlock[] = [];
public async updateOrphanedBlocks(): Promise<void> {
try {
this.chainTips = await bitcoinClient.getChainTips();
this.orphanedBlocks = [];
for (const chain of this.chainTips) {
if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
let block = await bitcoinClient.getBlock(chain.hash);
while (block && block.confirmations === -1) {
this.orphanedBlocks.push({
height: block.height,
hash: block.hash,
status: chain.status
});
block = await bitcoinClient.getBlock(block.previousblockhash);
}
}
}
logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
} catch (e) {
logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
}
}
public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
if (height === undefined) {
return [];
}
const orphans: OrphanedBlock[] = [];
for (const block of this.orphanedBlocks) {
if (block.height === height) {
orphans.push(block);
}
}
return orphans;
}
}
export default new ChainTips();

View File

@@ -1,4 +1,4 @@
import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
import { isIP } from 'net';
@@ -35,25 +35,16 @@ export class Common {
}
static getFeesInRange(transactions: TransactionExtended[], rangeLength: number) {
const filtered: TransactionExtended[] = [];
let lastValidRate = Infinity;
// filter out anomalous fee rates to ensure monotonic range
for (const tx of transactions) {
if (tx.effectiveFeePerVsize <= lastValidRate) {
filtered.push(tx);
lastValidRate = tx.effectiveFeePerVsize;
}
}
const arr = [filtered[filtered.length - 1].effectiveFeePerVsize];
const arr = [transactions[transactions.length - 1].effectiveFeePerVsize];
const chunk = 1 / (rangeLength - 1);
let itemsToAdd = rangeLength - 2;
while (itemsToAdd > 0) {
arr.push(filtered[Math.floor(filtered.length * chunk * itemsToAdd)].effectiveFeePerVsize);
arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].effectiveFeePerVsize);
itemsToAdd--;
}
arr.push(filtered[0].effectiveFeePerVsize);
arr.push(transactions[0].effectiveFeePerVsize);
return arr;
}
@@ -164,30 +155,6 @@ export class Common {
return parents;
}
// calculates the ratio of matched transactions to projected transactions by weight
static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number {
let matchedWeight = 0;
let projectedWeight = 0;
const inBlock = {};
for (const tx of transactions) {
inBlock[tx.txid] = tx;
}
// look for transactions that were expected in the template, but missing from the mined block
for (const tx of projectedBlock.transactions) {
if (inBlock[tx.txid]) {
matchedWeight += tx.vsize * 4;
}
projectedWeight += tx.vsize * 4;
}
projectedWeight += transactions[0].weight;
matchedWeight += transactions[0].weight;
return projectedWeight ? matchedWeight / projectedWeight : 1;
}
static getSqlInterval(interval: string | null): string | null {
switch (interval) {
case '24h': return '1 DAY';
@@ -199,7 +166,6 @@ export class Common {
case '1y': return '1 YEAR';
case '2y': return '2 YEAR';
case '3y': return '3 YEAR';
case '4y': return '4 YEAR';
default: return null;
}
}
@@ -262,21 +228,14 @@ export class Common {
].join('x');
}
static utcDateToMysql(date?: number | null): string | null {
if (date === null) {
return null;
}
static utcDateToMysql(date?: number): string {
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
static findSocketNetwork(addr: string): {network: string | null, url: string} {
let network: string | null = null;
let url: string = addr;
if (config.LIGHTNING.BACKEND === 'cln') {
url = addr.split('://')[1];
}
let url = addr.split('://')[1];
if (!url) {
return {
@@ -293,7 +252,7 @@ export class Common {
}
} else if (addr.indexOf('i2p') !== -1) {
network = 'i2p';
} else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
} else if (addr.indexOf('ipv4') !== -1) {
const ipv = isIP(url.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
@@ -303,7 +262,7 @@ export class Common {
url: addr,
};
}
} else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
} else if (addr.indexOf('ipv6') !== -1) {
url = url.split('[')[1].split(']')[0];
const ipv = isIP(url);
if (ipv === 6) {

View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 59;
private static currentVersion = 52;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -62,8 +62,8 @@ class DatabaseMigration {
if (databaseSchemaVersion <= 2) {
// Disable some spam logs when they're not relevant
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
this.uniqueLogs.push(this.blocksTruncatedMessage);
this.uniqueLogs.push(this.hashratesTruncatedMessage);
}
logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -86,7 +86,7 @@ class DatabaseMigration {
try {
await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
if (databaseSchemaVersion === 0) {
logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
} else {
logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
}
@@ -300,7 +300,7 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.updateToSchemaVersion(27);
}
if (databaseSchemaVersion < 28 && isBitcoin === true) {
if (config.LIGHTNING.ENABLED) {
this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
@@ -461,60 +461,13 @@ class DatabaseMigration {
await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions'));
try {
await this.$convertCompactCpfpTables();
await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
await this.updateToSchemaVersion(52);
} catch (e) {
} catch(e) {
logger.warn('' + (e instanceof Error ? e.message : e));
}
}
if (databaseSchemaVersion < 53) {
await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
await this.updateToSchemaVersion(53);
}
if (databaseSchemaVersion < 54) {
this.uniqueLog(logger.notice, `'prices' table has been truncated`);
await this.$executeQuery(`TRUNCATE prices`);
if (isBitcoin === true) {
this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
await this.$executeQuery(`TRUNCATE blocks_prices`);
}
await this.updateToSchemaVersion(54);
}
if (databaseSchemaVersion < 55) {
await this.$executeQuery(this.getAdditionalBlocksDataQuery());
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.updateToSchemaVersion(55);
}
if (databaseSchemaVersion < 56) {
await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
await this.$executeQuery('TRUNCATE TABLE `blocks`');
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
this.uniqueLog(logger.notice, '`pools` table has been truncated`');
await this.updateToSchemaVersion(56);
}
if (databaseSchemaVersion < 57 && isBitcoin === true) {
await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
await this.updateToSchemaVersion(57);
}
if (databaseSchemaVersion < 58) {
// We only run some migration queries for this version
await this.updateToSchemaVersion(58);
}
if (databaseSchemaVersion < 59 && (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet')) {
// https://github.com/mempool/mempool/issues/3360
await this.$executeQuery(`TRUNCATE prices`);
}
}
/**
@@ -638,15 +591,10 @@ class DatabaseMigration {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
}
if (version < 9 && isBitcoin === true) {
if (version < 9 && isBitcoin === true) {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
}
if (version < 58) {
queries.push(`DELETE FROM state WHERE name = 'last_hashrates_indexing'`);
queries.push(`DELETE FROM state WHERE name = 'last_weekly_hashrates_indexing'`);
}
return queries;
}
@@ -793,28 +741,6 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getAdditionalBlocksDataQuery(): string {
return `ALTER TABLE blocks
ADD median_timestamp timestamp NOT NULL,
ADD coinbase_address varchar(100) NULL,
ADD coinbase_signature varchar(500) NULL,
ADD coinbase_signature_ascii varchar(500) NULL,
ADD avg_tx_size double unsigned NOT NULL,
ADD total_inputs int unsigned NOT NULL,
ADD total_outputs int unsigned NOT NULL,
ADD total_output_amt bigint unsigned NOT NULL,
ADD fee_percentiles longtext NULL,
ADD median_fee_amt int unsigned NULL,
ADD segwit_total_txs int unsigned NOT NULL,
ADD segwit_total_size int unsigned NOT NULL,
ADD segwit_total_weight int unsigned NOT NULL,
ADD header varchar(160) NOT NULL,
ADD utxoset_change int NOT NULL,
ADD utxoset_size int unsigned NULL,
ADD total_input_amt bigint unsigned NULL
`;
}
private getCreateDailyStatsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS hashrates (
hashrate_timestamp timestamp NOT NULL,
@@ -1032,16 +958,25 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $blocksReindexingTruncate(): Promise<void> {
logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
await Common.sleep$(5000);
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];
await this.$executeQuery(`TRUNCATE blocks`);
await this.$executeQuery(`TRUNCATE hashrates`);
await this.$executeQuery(`TRUNCATE difficulty_adjustments`);
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {
logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
continue;
}
await this.$executeQuery(`TRUNCATE ${table}`, true);
if (table === 'hashrates') {
await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
}
logger.notice(`Table ${table} has been truncated`);
}
} catch (e) {
logger.warn(`Unable to erase indexed data`);
}
}
private async $convertCompactCpfpTables(): Promise<void> {

View File

@@ -9,11 +9,9 @@ export interface DifficultyAdjustment {
remainingBlocks: number; // Block count
remainingTime: number; // Duration of time in ms
previousRetarget: number; // Percent: -75 to 300
previousTime: number; // Unix time in ms
nextRetargetHeight: number; // Block Height
timeAvg: number; // Duration of time in ms
timeOffset: number; // (Testnet) Time since last block (cap @ 20min) in ms
expectedBlocks: number; // Block count
}
export function calcDifficultyAdjustment(
@@ -34,12 +32,12 @@ export function calcDifficultyAdjustment(
const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
let difficultyChange = 0;
let timeAvgSecs = diffSeconds / blocksInEpoch;
let timeAvgSecs = BLOCK_SECONDS_TARGET;
// Only calculate the estimate once we have 7.2% of blocks in current epoch
if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
timeAvgSecs = diffSeconds / blocksInEpoch;
difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
// Max increase is x4 (+300%)
if (difficultyChange > 300) {
@@ -76,11 +74,9 @@ export function calcDifficultyAdjustment(
remainingBlocks,
remainingTime,
previousRetarget,
previousTime: DATime,
nextRetargetHeight,
timeAvg,
timeOffset,
expectedBlocks,
};
}

View File

@@ -9,35 +9,25 @@ import { TransactionExtended } from '../mempool.interfaces';
import { Common } from './common';
class DiskCache {
private cacheSchemaVersion = 3;
private cacheSchemaVersion = 1;
private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-cache.json';
private static TMP_FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/tmp-cache{number}.json';
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
private static CHUNK_FILES = 25;
private isWritingCache = false;
constructor() {
if (!cluster.isPrimary) {
return;
}
process.on('SIGINT', (e) => {
this.$saveCacheToDisk(true);
process.exit(0);
});
}
constructor() { }
async $saveCacheToDisk(sync: boolean = false): Promise<void> {
async $saveCacheToDisk(): Promise<void> {
if (!cluster.isPrimary) {
return;
}
if (this.isWritingCache) {
logger.debug('Saving cache already in progress. Skipping.');
logger.debug('Saving cache already in progress. Skipping.')
return;
}
try {
logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
logger.debug('Writing mempool and blocks data to disk cache (async)...');
this.isWritingCache = true;
const mempool = memPool.getMempool();
@@ -50,48 +40,19 @@ class DiskCache {
const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
if (sync) {
fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
}
fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
} else {
await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
network: config.MEMPOOL.NETWORK,
cacheSchemaVersion: this.cacheSchemaVersion,
blocks: blocks.getBlocks(),
blockSummaries: blocks.getBlockSummaries(),
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), { flag: 'w' });
}
await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
}
logger.debug('Mempool and blocks data saved to disk cache');
this.isWritingCache = false;
} catch (e) {
@@ -100,29 +61,14 @@ class DiskCache {
}
}
wipeCache(): void {
logger.notice(`Wiping nodejs backend cache/cache*.json files`);
try {
fs.unlinkSync(DiskCache.FILE_NAME);
} catch (e: any) {
if (e?.code !== 'ENOENT') {
logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
}
}
wipeCache() {
fs.unlinkSync(DiskCache.FILE_NAME);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
try {
fs.unlinkSync(filename);
} catch (e: any) {
if (e?.code !== 'ENOENT') {
logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
}
}
fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
}
loadMempoolCache(): void {
loadMempoolCache() {
if (!fs.existsSync(DiskCache.FILE_NAME)) {
return;
}
@@ -136,10 +82,6 @@ class DiskCache {
logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
return this.wipeCache();
}
if (data.network && data.network !== config.MEMPOOL.NETWORK) {
logger.notice('Disk cache contains data from a different network. Clearing it and skipping the cache loading.');
return this.wipeCache();
}
if (data.mempoolArray) {
for (const tx of data.mempoolArray) {

View File

@@ -559,17 +559,6 @@ class ChannelsApi {
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
// https://github.com/mempool/mempool/issues/3006
if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
channel.last_update = null;
}
if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
policy1.last_update = null;
}
if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
policy2.last_update = null;
}
const query = `INSERT INTO channels
(
id,

View File

@@ -228,7 +228,7 @@ class NodesApi {
nodes.capacity
FROM nodes
ORDER BY capacity DESC
LIMIT 6
LIMIT 100
`;
[rows] = await DB.query(query);
@@ -269,26 +269,14 @@ class NodesApi {
let query: string;
if (full === false) {
query = `
SELECT
nodes.public_key as publicKey,
IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
nodes.channels,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
nodes.channels
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
ORDER BY channels DESC
LIMIT 6;
LIMIT 100;
`;
[rows] = await DB.query(query);
for (let i = 0; i < rows.length; ++i) {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);
}
} else {
query = `
SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
@@ -374,13 +362,7 @@ class NodesApi {
public async $searchNodeByPublicKeyOrAlias(search: string) {
try {
const publicKeySearch = search.replace('%', '') + '%';
const aliasSearch = search
.replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".
.replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
.split(' ')
.filter(key => key.length)
.map((search) => '+' + search + '*').join(' ');
// %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
return rows;
@@ -417,24 +399,24 @@ class NodesApi {
if (!ispList[isp1]) {
ispList[isp1] = {
ids: [channel.isp1ID],
id: channel.isp1ID.toString(),
capacity: 0,
channels: 0,
nodes: {},
};
} else if (ispList[isp1].ids.includes(channel.isp1ID) === false) {
ispList[isp1].ids.push(channel.isp1ID);
} else if (ispList[isp1].id.indexOf(channel.isp1ID) === -1) {
ispList[isp1].id += ',' + channel.isp1ID.toString();
}
if (!ispList[isp2]) {
ispList[isp2] = {
ids: [channel.isp2ID],
id: channel.isp2ID.toString(),
capacity: 0,
channels: 0,
nodes: {},
};
} else if (ispList[isp2].ids.includes(channel.isp2ID) === false) {
ispList[isp2].ids.push(channel.isp2ID);
} else if (ispList[isp2].id.indexOf(channel.isp2ID) === -1) {
ispList[isp2].id += ',' + channel.isp2ID.toString();
}
ispList[isp1].capacity += channel.capacity;
@@ -444,11 +426,11 @@ class NodesApi {
ispList[isp2].channels += 1;
ispList[isp2].nodes[channel.node2PublicKey] = true;
}
const ispRanking: any[] = [];
for (const isp of Object.keys(ispList)) {
ispRanking.push([
ispList[isp].ids.sort((a, b) => a - b).join(','),
ispList[isp].id,
isp,
ispList[isp].capacity,
ispList[isp].channels,
@@ -642,11 +624,6 @@ class NodesApi {
*/
public async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
// https://github.com/mempool/mempool/issues/3006
if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
node.last_update = null;
}
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,

View File

@@ -0,0 +1,123 @@
import logger from '../logger';
import * as http from 'http';
import * as https from 'https';
import axios, { AxiosResponse } from 'axios';
import { IConversionRates } from '../mempool.interfaces';
import config from '../config';
import backendInfo from './backend-info';
import { SocksProxyAgent } from 'socks-proxy-agent';
class FiatConversion {
private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
private conversionRates: IConversionRates = {};
private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
public ratesInitialized = false; // If true, it means rates are ready for use
constructor() {
for (const fiat of this.debasingFiatCurrencies) {
this.conversionRates[fiat] = 0;
}
}
public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
this.ratesChangedCallback = fn;
}
public startService() {
const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
logger.info('Starting currency rates service');
if (config.SOCKS5PROXY.ENABLED) {
logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
} else {
logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
}
setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
this.updateCurrency();
}
public getConversionRates() {
return this.conversionRates;
}
private async updateCurrency(): Promise<void> {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpAgent?: http.Agent;
httpsAgent?: https.Agent;
}
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0;
while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try {
if (config.SOCKS5PROXY.ENABLED) {
let socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
}
// Handle proxy agent for onion addresses
if (isHTTP) {
axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
} else {
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
}
logger.debug('Querying currency rates service...');
const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
if (response.statusText === 'error' || !response.data) {
throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
}
for (const rate of response.data.data) {
if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
}
}
this.ratesInitialized = true;
logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.conversionRates);
}
break;
} catch (e) {
logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
retry++;
}
}
}
}
export default new FiatConversion();

View File

@@ -55,12 +55,11 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
clChannelsDict[clChannel.short_channel_id] = clChannel;
clChannelsDictCount[clChannel.short_channel_id] = 1;
} else {
const fullChannel = await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]);
if (fullChannel !== null) {
consolidatedChannelList.push(fullChannel);
delete clChannelsDict[clChannel.short_channel_id];
clChannelsDictCount[clChannel.short_channel_id]++;
}
consolidatedChannelList.push(
await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id])
);
delete clChannelsDict[clChannel.short_channel_id];
clChannelsDictCount[clChannel.short_channel_id]++;
}
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
@@ -75,10 +74,7 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
channelProcessed = 0;
const keys = Object.keys(clChannelsDict);
for (const short_channel_id of keys) {
const incompleteChannel = await buildIncompleteChannel(clChannelsDict[short_channel_id]);
if (incompleteChannel !== null) {
consolidatedChannelList.push(incompleteChannel);
}
consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
@@ -96,13 +92,10 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
* Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
* In this case, clightning knows the channel policy for both nodes
*/
async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel | null> {
async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel> {
const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
if (!tx) {
return null;
}
const parts = clChannelA.short_channel_id.split('x');
const outputIdx = parts[2];
@@ -122,11 +115,8 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILigh
* Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
* In this case, clightning knows the channel policy of only one node
*/
async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel | null> {
async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel> {
const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
if (!tx) {
return null;
}
const parts = clChannel.short_channel_id.split('x');
const outputIdx = parts[2];

View File

@@ -21,7 +21,7 @@ export namespace ILightningApi {
export interface Channel {
channel_id: string;
chan_point: string;
last_update: number | null;
last_update: number;
node1_pub: string;
node2_pub: string;
capacity: string;
@@ -36,11 +36,11 @@ export namespace ILightningApi {
fee_rate_milli_msat: string;
disabled: boolean;
max_htlc_msat: string;
last_update: number | null;
last_update: number;
}
export interface Node {
last_update: number | null;
last_update: number;
pub_key: string;
alias: string;
addresses: {

View File

@@ -17,7 +17,7 @@ class LndApi implements AbstractLightningApi {
httpsAgent: new Agent({
ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
}),
timeout: config.LND.TIMEOUT
timeout: 10000
};
}
}

View File

@@ -33,7 +33,7 @@ class MempoolBlocks {
return this.mempoolBlockDeltas;
}
public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
const latestMempool = memPool;
const memPoolArray: TransactionExtended[] = [];
for (const i in latestMempool) {
@@ -75,14 +75,10 @@ class MempoolBlocks {
logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
if (saveResults) {
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
this.mempoolBlocks = blocks;
this.mempoolBlockDeltas = deltas;
}
return blocks;
this.mempoolBlocks = blocks;
this.mempoolBlockDeltas = deltas;
}
private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
@@ -97,14 +93,14 @@ class MempoolBlocks {
blockSize += tx.size;
transactions.push(tx);
} else {
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
blockWeight = tx.weight;
blockSize = tx.size;
transactions = [tx];
}
});
if (transactions.length) {
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
}
return mempoolBlocks;
@@ -147,7 +143,7 @@ class MempoolBlocks {
return mempoolBlockDeltas;
}
public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }): Promise<void> {
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
const strippedMempool: { [txid: string]: ThreadTransaction } = {};
@@ -188,21 +184,19 @@ class MempoolBlocks {
this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
const { blocks, clusters } = await workerResultPromise;
this.processBlockTemplates(newMempool, blocks, clusters);
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
return this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
} catch (e) {
logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
}
return this.mempoolBlocks;
}
public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[]): Promise<void> {
if (!this.txSelectionWorker) {
// need to reset the worker
this.makeBlockTemplates(newMempool, saveResults);
return;
return this.makeBlockTemplates(newMempool);
}
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
@@ -230,16 +224,16 @@ class MempoolBlocks {
this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
const { blocks, clusters } = await workerResultPromise;
this.processBlockTemplates(newMempool, blocks, clusters);
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
} catch (e) {
logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
}
}
private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] {
private processBlockTemplates(mempool, blocks, clusters): void {
// update this thread's mempool with the results
blocks.forEach(block => {
block.forEach(tx => {
@@ -255,7 +249,7 @@ class MempoolBlocks {
cluster.forEach(txid => {
if (txid === tx.txid) {
matched = true;
} else {
} else if (mempool[txid]) {
const relative = {
txid: txid,
fee: mempool[txid].fee,
@@ -281,29 +275,27 @@ class MempoolBlocks {
const mempoolBlocks = blocks.map((transactions, blockIndex) => {
return this.dataToMempoolBlocks(transactions.map(tx => {
return mempool[tx.txid] || null;
}).filter(tx => !!tx), blockIndex);
}).filter(tx => !!tx), undefined, undefined, blockIndex);
});
if (saveResults) {
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
this.mempoolBlocks = mempoolBlocks;
this.mempoolBlockDeltas = deltas;
}
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
return mempoolBlocks;
this.mempoolBlocks = mempoolBlocks;
this.mempoolBlockDeltas = deltas;
}
private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
let totalSize = 0;
let totalWeight = 0;
const fitTransactions: TransactionExtended[] = [];
transactions.forEach(tx => {
totalSize += tx.size;
totalWeight += tx.weight;
if ((totalWeight + tx.weight) <= config.MEMPOOL.BLOCK_WEIGHT_UNITS * 1.2) {
fitTransactions.push(tx);
}
});
private dataToMempoolBlocks(transactions: TransactionExtended[],
blockSize: number | undefined, blockWeight: number | undefined, blocksIndex: number): MempoolBlockWithTransactions {
let totalSize = blockSize || 0;
let totalWeight = blockWeight || 0;
if (blockSize === undefined && blockWeight === undefined) {
totalSize = 0;
totalWeight = 0;
transactions.forEach(tx => {
totalSize += tx.size;
totalWeight += tx.weight;
});
}
let rangeLength = 4;
if (blocksIndex === 0) {
rangeLength = 8;
@@ -321,7 +313,7 @@ class MempoolBlocks {
medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
feeRange: Common.getFeesInRange(transactions, rangeLength),
transactionIds: transactions.map((tx) => tx.txid),
transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
transactions: transactions.map((tx) => Common.stripTransaction(tx)),
};
}
}

View File

@@ -10,33 +10,32 @@ import bitcoinClient from './bitcoin/bitcoin-client';
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import rbfCache from './rbf-cache';
class Mempool {
private static WEBSOCKET_REFRESH_RATE_MS = 10000;
private static LAZY_DELETE_AFTER_SECONDS = 30;
private inSync: boolean = false;
private mempoolCacheDelta: number = -1;
private mempoolCache: { [txId: string]: TransactionExtended } = {};
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
export class Mempool {
protected static WEBSOCKET_REFRESH_RATE_MS = 10000;
protected static LAZY_DELETE_AFTER_SECONDS = 30;
protected inSync: boolean = false;
protected mempoolCacheDelta: number = -1;
protected mempoolCache: { [txId: string]: TransactionExtended } = {};
protected mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
protected mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
deletedTransactions: TransactionExtended[]) => void) | undefined;
private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
protected asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
deletedTransactions: TransactionExtended[]) => Promise<void>) | undefined;
private txPerSecondArray: number[] = [];
private txPerSecond: number = 0;
protected txPerSecondArray: number[] = [];
protected txPerSecond: number = 0;
private vBytesPerSecondArray: VbytesPerSecond[] = [];
private vBytesPerSecond: number = 0;
private mempoolProtection = 0;
private latestTransactions: any[] = [];
private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100;
private SAMPLE_TIME = 10000; // In ms
private timer = new Date().getTime();
private missingTxCount = 0;
protected vBytesPerSecondArray: VbytesPerSecond[] = [];
protected vBytesPerSecond: number = 0;
protected mempoolProtection = 0;
protected latestTransactions: any[] = [];
constructor() {
this.init();
}
protected init(): void {
setInterval(this.updateTxPerSecond.bind(this), 1000);
setInterval(this.deleteExpiredTransactions.bind(this), 20000);
}
@@ -133,16 +132,6 @@ class Mempool {
loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
}
// https://github.com/mempool/mempool/issues/3283
const logEsplora404 = (missingTxCount, threshold, time) => {
const log = `In the past ${time / 1000} seconds, esplora tx API replied ${missingTxCount} times with a 404 error code while updating nodejs backend mempool`;
if (missingTxCount >= threshold) {
logger.warn(log);
} else if (missingTxCount > 0) {
logger.debug(log);
}
};
for (const txid of transactions) {
if (!this.mempoolCache[txid]) {
try {
@@ -157,10 +146,7 @@ class Mempool {
}
hasChange = true;
newTransactions.push(transaction);
} catch (e: any) {
if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
this.missingTxCount++;
}
} catch (e) {
logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
}
}
@@ -170,14 +156,6 @@ class Mempool {
}
}
// Reset esplora 404 counter and log a warning if needed
const elapsedTime = new Date().getTime() - this.timer;
if (elapsedTime > this.SAMPLE_TIME) {
logEsplora404(this.missingTxCount, this.ESPLORA_MISSING_TX_WARNING_THRESHOLD, elapsedTime);
this.timer = new Date().getTime();
this.missingTxCount = 0;
}
// Prevent mempool from clear on bitcoind restart by delaying the deletion
if (this.mempoolProtection === 0
&& currentMempoolSize > 20000
@@ -243,7 +221,7 @@ class Mempool {
}
}
private updateTxPerSecond() {
protected updateTxPerSecond() {
const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);
this.txPerSecond = this.txPerSecondArray.length / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD || 0;
@@ -256,7 +234,7 @@ class Mempool {
}
}
private deleteExpiredTransactions() {
protected deleteExpiredTransactions() {
const now = new Date().getTime();
for (const tx in this.mempoolCache) {
const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
@@ -267,7 +245,7 @@ class Mempool {
}
}
private $getMempoolInfo() {
protected $getMempoolInfo() {
if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
return Promise.all([
bitcoinClient.getMempoolInfo(),

View File

@@ -1,13 +1,13 @@
import { Application, Request, Response } from 'express';
import config from "../../config";
import logger from '../../logger';
import audits from '../audit';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import BlocksRepository from '../../repositories/BlocksRepository';
import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
import HashratesRepository from '../../repositories/HashratesRepository';
import bitcoinClient from '../bitcoin/bitcoin-client';
import mining from "./mining";
import PricesRepository from '../../repositories/PricesRepository';
class MiningRoutes {
public initRoutes(app: Application) {
@@ -32,27 +32,9 @@ class MiningRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
.get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
;
}
private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
try {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
if (req.query.timestamp) {
res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
parseInt(<string>req.query.timestamp ?? 0, 10)
));
} else {
res.status(200).send(await PricesRepository.$getHistoricalPrices());
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getPool(req: Request, res: Response): Promise<void> {
try {
const stats = await mining.$getPoolStat(req.params.slug);
@@ -263,7 +245,7 @@ class MiningRoutes {
const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
if (!audit) {
res.status(204).send(`This block has not been audited.`);
res.status(404).send(`This block has not been audited.`);
return;
}

View File

@@ -11,14 +11,12 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
import config from '../../config';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository';
import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
import { IEsploraApi } from '../bitcoin/esplora-api.interface';
import database from '../../database';
class Mining {
private blocksPriceIndexingRunning = false;
public lastHashrateIndexingDate: number | null = null;
public lastWeeklyHashrateIndexingDate: number | null = null;
blocksPriceIndexingRunning = false;
constructor() {
}
/**
* Get historical block predictions match rate
@@ -102,7 +100,6 @@ class Mining {
rank: rank++,
emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
slug: poolInfo.slug,
avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
};
poolsStats.push(poolStat);
});
@@ -118,7 +115,7 @@ class Mining {
poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
} catch (e) {
poolsStatistics['lastEstimatedHashrate'] = 0;
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
}
return poolsStatistics;
@@ -142,14 +139,11 @@ class Mining {
const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);
const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);
let currentEstimatedHashrate = 0;
try {
currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
} catch (e) {
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
}
return {
@@ -166,8 +160,6 @@ class Mining {
},
estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
reportedHashrate: null,
avgBlockHealth: avgHealth,
totalReward: totalReward,
};
}
@@ -179,26 +171,25 @@ class Mining {
}
/**
* Generate weekly mining pool hashrate history
* [INDEXING] Generate weekly mining pool hashrate history
*/
public async $generatePoolHashrateHistory(): Promise<void> {
const now = new Date();
const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
// Run only if:
// * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg)
// * lastestRunDate is set to 0 (node backend restart, reorg)
// * we started a new week (around Monday midnight)
const runIndexing = this.lastWeeklyHashrateIndexingDate === null ||
now.getUTCDay() === 1 && this.lastWeeklyHashrateIndexingDate !== now.getUTCDate();
const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
if (!runIndexing) {
logger.debug(`Pool hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
return;
}
try {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.time * 1000;
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
const hashrates: any[] = [];
@@ -214,7 +205,7 @@ class Mining {
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
logger.debug(`Indexing weekly mining pool hashrate`, logger.tags.mining);
logger.debug(`Indexing weekly mining pool hashrate`);
loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -251,7 +242,7 @@ class Mining {
});
}
newlyIndexed += hashrates.length / Math.max(1, pools.length);
newlyIndexed += hashrates.length;
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
}
@@ -262,7 +253,7 @@ class Mining {
const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
const formattedDate = new Date(fromTimestamp).toUTCString();
logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
timer = new Date().getTime() / 1000;
indexedThisRun = 0;
loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
@@ -272,36 +263,36 @@ class Mining {
++indexedThisRun;
++totalIndexed;
}
this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate();
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
if (newlyIndexed > 0) {
logger.info(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining);
} else {
logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining);
}
loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
} catch (e) {
loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
throw e;
}
}
/**
* Generate daily hashrate data
* [INDEXING] Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// We only run this once a day around midnight
const today = new Date().getUTCDate();
if (today === this.lastHashrateIndexingDate) {
logger.debug(`Network hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
const now = new Date().getUTCDate();
if (now === latestRunDate) {
return;
}
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
try {
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.time * 1000;
const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round(lastMidnight.getTime());
@@ -314,7 +305,7 @@ class Mining {
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
logger.debug(`Indexing daily network hashrate`, logger.tags.mining);
logger.debug(`Indexing daily network hashrate`);
loadingIndicators.setProgress('daily-hashrate-indexing', 0);
while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -352,7 +343,7 @@ class Mining {
const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
const formattedDate = new Date(fromTimestamp).toUTCString();
logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
timer = new Date().getTime() / 1000;
indexedThisRun = 0;
loadingIndicators.setProgress('daily-hashrate-indexing', progress);
@@ -377,16 +368,16 @@ class Mining {
newlyIndexed += hashrates.length;
await HashratesRepository.$saveHashrates(hashrates);
this.lastHashrateIndexingDate = new Date().getUTCDate();
await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
if (newlyIndexed > 0) {
logger.info(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
} else {
logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
}
loadingIndicators.setProgress('daily-hashrate-indexing', 100);
} catch (e) {
loadingIndicators.setProgress('daily-hashrate-indexing', 100);
logger.err(`Daily network hashrate indexing failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
throw e;
}
}
@@ -402,13 +393,13 @@ class Mining {
}
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
let currentDifficulty = genesisBlock.difficulty;
let totalIndexed = 0;
if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: genesisBlock.timestamp,
time: genesisBlock.time,
height: 0,
difficulty: currentDifficulty,
adjustment: 0.0,
@@ -452,13 +443,13 @@ class Mining {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
if (elapsedSeconds > 5) {
const progress = Math.round(totalBlockChecked / blocks.length * 100);
logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`);
timer = new Date().getTime() / 1000;
}
}
if (totalIndexed > 0) {
logger.info(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
logger.notice(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
} else {
logger.debug(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
}
@@ -467,7 +458,7 @@ class Mining {
/**
* Create a link between blocks and the latest price at when they were mined
*/
public async $indexBlockPrices(): Promise<void> {
public async $indexBlockPrices() {
if (this.blocksPriceIndexingRunning === true) {
return;
}
@@ -505,7 +496,7 @@ class Mining {
if (blocksWithoutPrices.length > 200000) {
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr, logger.tags.mining);
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0;
}
@@ -517,7 +508,7 @@ class Mining {
if (blocksWithoutPrices.length > 200000) {
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr, logger.tags.mining);
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
}
} catch (e) {
@@ -528,41 +519,6 @@ class Mining {
this.blocksPriceIndexingRunning = false;
}
/**
* Index core coinstatsindex
*/
public async $indexCoinStatsIndex(): Promise<void> {
let timer = new Date().getTime() / 1000;
let totalIndexed = 0;
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
let currentBlockHeight = blockchainInfo.blocks;
while (currentBlockHeight > 0) {
const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
currentBlockHeight, currentBlockHeight - 10000);
for (const block of indexedBlocks) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
Math.round(txoutset.block_info.prevout_spent * 100000000));
++totalIndexed;
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
if (elapsedSeconds > 5) {
logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
timer = new Date().getTime() / 1000;
}
}
currentBlockHeight -= 10000;
}
if (totalIndexed) {
logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
}
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);
@@ -574,7 +530,6 @@ class Mining {
private getTimeRange(interval: string | null, scale = 1): number {
switch (interval) {
case '4y': return 43200 * scale; // 12h
case '3y': return 43200 * scale; // 12h
case '2y': return 28800 * scale; // 8h
case '1y': return 28800 * scale; // 8h

View File

@@ -1,161 +1,289 @@
import DB from '../database';
import logger from '../logger';
import config from '../config';
import PoolsRepository from '../repositories/PoolsRepository';
import { PoolTag } from '../mempool.interfaces';
import diskCache from './disk-cache';
import BlocksRepository from '../repositories/BlocksRepository';
interface Pool {
name: string;
link: string;
regexes: string[];
addresses: string[];
slug: string;
}
class PoolsParser {
miningPools: any[] = [];
unknownPool: any = {
'id': 0,
'name': 'Unknown',
'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
'regexes': '[]',
'addresses': '[]',
'slug': 'unknown'
};
private uniqueLogs: string[] = [];
private uniqueLog(loggerFunction: any, msg: string): void {
if (this.uniqueLogs.includes(msg)) {
return;
}
this.uniqueLogs.push(msg);
loggerFunction(msg);
}
public setMiningPools(pools): void {
for (const pool of pools) {
pool.regexes = pool.tags;
pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
delete(pool.tags);
}
this.miningPools = pools;
}
slugWarnFlag = false;
/**
* Populate our db with updated mining pool definition
* @param pools
* Parse the pools.json file, consolidate the data and dump it into the database
*/
public async migratePoolsJson(): Promise<void> {
// We also need to wipe the backend cache to make sure we don't serve blocks with
// the wrong mining pool (usually happen with unknown blocks)
diskCache.wipeCache();
public async migratePoolsJson(poolsJson: object): Promise<void> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
await this.$insertUnknownPool();
// First we save every entries without paying attention to pool duplication
const poolsDuplicated: Pool[] = [];
for (const pool of this.miningPools) {
if (!pool.id) {
logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
continue;
const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
for (let i = 0; i < coinbaseTags.length; ++i) {
poolsDuplicated.push({
'name': (<Pool>coinbaseTags[i][1]).name,
'link': (<Pool>coinbaseTags[i][1]).link,
'regexes': [coinbaseTags[i][0]],
'addresses': [],
'slug': ''
});
}
const addressesTags = Object.entries(poolsJson['payout_addresses']);
for (let i = 0; i < addressesTags.length; ++i) {
poolsDuplicated.push({
'name': (<Pool>addressesTags[i][1]).name,
'link': (<Pool>addressesTags[i][1]).link,
'regexes': [],
'addresses': [addressesTags[i][0]],
'slug': ''
});
}
// Then, we find unique mining pool names
const poolNames: string[] = [];
for (let i = 0; i < poolsDuplicated.length; ++i) {
if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
poolNames.push(poolsDuplicated[i].name);
}
}
logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);
// Get existing pools from the db
let existingPools;
try {
if (config.DATABASE.ENABLED === true) {
[existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
} else {
existingPools = [];
}
} catch (e) {
logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
return;
}
this.miningPools = [];
// Finally, we generate the final consolidated pools data
const finalPoolDataAdd: Pool[] = [];
const finalPoolDataUpdate: Pool[] = [];
const finalPoolDataRename: Pool[] = [];
for (let i = 0; i < poolNames.length; ++i) {
let allAddresses: string[] = [];
let allRegexes: string[] = [];
const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
for (let y = 0; y < match.length; ++y) {
allAddresses = allAddresses.concat(match[y].addresses);
allRegexes = allRegexes.concat(match[y].regexes);
}
const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
if (!poolDB) {
// New mining pool
const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.debug(`Inserting new mining pool ${pool.name}`);
await PoolsRepository.$insertNewMiningPool(pool, slug);
await this.$deleteUnknownBlocks();
const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
let slug: string | undefined;
try {
slug = poolsJson['slugs'][poolNames[i]];
} catch (e) {
if (this.slugWarnFlag === false) {
logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
this.slugWarnFlag = true;
}
}
if (slug === undefined) {
// Only keep alphanumerical
slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
}
const poolObj = {
'name': finalPoolName,
'link': match[0].link,
'regexes': allRegexes,
'addresses': allAddresses,
'slug': slug
};
const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
if (existingPool !== undefined) {
// Check if any data was actually updated
const equals = (a, b) =>
a.length === b.length &&
a.every((v, i) => v === b[i]);
if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
finalPoolDataUpdate.push(poolObj);
}
} else {
if (poolDB.name !== pool.name) {
// Pool has been renamed
const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
// Double check that if we're not just renaming a pool (same address same regex)
const [poolToRename]: any[] = await DB.query(`
SELECT * FROM pools
WHERE addresses = ? OR regexes = ?`,
[JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
);
if (poolToRename && poolToRename.length > 0) {
// We're actually renaming an existing pool
finalPoolDataRename.push({
'name': poolObj.name,
'link': poolObj.link,
'regexes': allRegexes,
'addresses': allAddresses,
'slug': slug
});
logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
} else {
logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
finalPoolDataAdd.push(poolObj);
}
if (poolDB.link !== pool.link) {
// Pool link has changed
logger.debug(`Updating link for ${pool.name} mining pool`);
await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
}
this.miningPools.push({
'name': finalPoolName,
'link': match[0].link,
'regexes': JSON.stringify(allRegexes),
'addresses': JSON.stringify(allAddresses),
'slug': slug
});
}
if (config.DATABASE.ENABLED === false) { // Don't run db operations
logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
return;
}
if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
finalPoolDataRename.length > 0
) {
logger.debug(`Update pools table now`, logger.tags.mining);
// Add new mining pools into the database
let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
for (let i = 0; i < finalPoolDataAdd.length; ++i) {
queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
'${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
}
queryAdd = queryAdd.slice(0, -1) + ';';
// Updated existing mining pools in the database
const updateQueries: string[] = [];
for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
updateQueries.push(`
UPDATE pools
SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
slug='${finalPoolDataUpdate[i].slug}'
WHERE name='${finalPoolDataUpdate[i].name}'
;`);
}
// Rename mining pools
const renameQueries: string[] = [];
for (let i = 0; i < finalPoolDataRename.length; ++i) {
renameQueries.push(`
UPDATE pools
SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
slug='${finalPoolDataRename[i].slug}'
WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
;`);
}
try {
if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
await this.$deleteBlocskToReindex(finalPoolDataUpdate);
}
if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
JSON.stringify(pool.regexes) !== poolDB.regexes) {
// Pool addresses changed or coinbase tags changed
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
await this.$deleteBlocksForPool(poolDB);
if (finalPoolDataAdd.length > 0) {
await DB.query({ sql: queryAdd, timeout: 120000 });
}
for (const query of updateQueries) {
await DB.query({ sql: query, timeout: 120000 });
}
for (const query of renameQueries) {
await DB.query({ sql: query, timeout: 120000 });
}
await this.insertUnknownPool();
logger.info('Mining pools.json import completed', logger.tags.mining);
} catch (e) {
logger.err(`Cannot import pools in the database`, logger.tags.mining);
throw e;
}
}
logger.info('Mining pools-v2.json import completed');
try {
await this.insertUnknownPool();
} catch (e) {
logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
throw e;
}
}
/**
* Manually add the 'unknown pool'
*/
public async $insertUnknownPool(): Promise<void> {
if (!config.DATABASE.ENABLED) {
return;
}
private async insertUnknownPool() {
try {
const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
if (rows.length === 0) {
await DB.query({
sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
`});
} else {
await DB.query(`UPDATE pools
SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
regexes='[]', addresses='[]',
slug='${this.unknownPool.slug}',
unique_id=0
WHERE slug='${this.unknownPool.slug}'
slug='unknown'
WHERE name='Unknown'
`);
}
} catch (e) {
logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
}
}
/**
* Delete indexed blocks for an updated mining pool
*
* @param pool
* Delete blocks which needs to be reindexed
*/
private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
return;
}
// Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
// Ignore early days of Bitcoin as there were no mining pool yet
const [oldestPoolBlock]: any[] = await DB.query(`
SELECT height
FROM blocks
WHERE pool_id = ?
ORDER BY height
LIMIT 1`,
[pool.id]
);
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
[unknownPool[0].id]
);
logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ?`,
[pool.id]
);
}
const blockCount = await BlocksRepository.$blockCount(null, null);
if (blockCount === 0) {
return;
}
private async $deleteUnknownBlocks(): Promise<void> {
for (const updatedPool of finalPoolDataUpdate) {
const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
if (pool.length > 0) {
logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
}
}
// Ignore early days of Bitcoin as there were not mining pool yet
logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ? AND height >= 130635`,
[unknownPool[0].id]
);
await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
await DB.query(`DELETE FROM hashrates`);
}
}

View File

@@ -375,17 +375,6 @@ class StatisticsApi {
}
}
public async $list4Y(): Promise<OptimizedStatistic[]> {
try {
const query = this.getQueryForDays(43200, '4 YEAR'); // 12h interval
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$list4Y() error' + (e instanceof Error ? e.message : e));
return [];
}
}
private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
return statistic.map((s) => {
return {

View File

@@ -14,11 +14,10 @@ class StatisticsRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', this.$getStatisticsByTime.bind(this, '1y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/4y', this.$getStatisticsByTime.bind(this, '4y'))
;
}
private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y', req: Request, res: Response) {
private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y', req: Request, res: Response) {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
@@ -55,9 +54,6 @@ class StatisticsRoutes {
case '3y':
result = await statisticsApi.$list3Y();
break;
case '4y':
result = await statisticsApi.$list4Y();
break;
default:
result = await statisticsApi.$list2H();
}

View File

@@ -14,7 +14,6 @@ class TransactionUtils {
vout: tx.vout
.map((vout) => ({
scriptpubkey_address: vout.scriptpubkey_address,
scriptpubkey_asm: vout.scriptpubkey_asm,
value: vout.value
}))
.filter((vout) => vout.value)

View File

@@ -1,13 +1,14 @@
import logger from '../logger';
import * as WebSocket from 'ws';
import {
BlockExtended, TransactionExtended, WebsocketResponse,
OptimizedStatistic, ILoadingIndicators
BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
OptimizedStatistic, ILoadingIndicators, IConversionRates
} from '../mempool.interfaces';
import blocks from './blocks';
import memPool from './mempool';
import backendInfo from './backend-info';
import mempoolBlocks from './mempool-blocks';
import fiatConversion from './fiat-conversion';
import { Common } from './common';
import loadingIndicators from './loading-indicators';
import config from '../config';
@@ -18,9 +19,6 @@ import feeApi from './fee-api';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import Audit from './audit';
import { deepClone } from '../utils/clone';
import priceUpdater from '../tasks/price-updater';
import { ApiPrice } from '../repositories/PricesRepository';
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
@@ -194,7 +192,7 @@ class WebsocketHandler {
});
}
handleNewConversionRates(conversionRates: ApiPrice) {
handleNewConversionRates(conversionRates: IConversionRates) {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
@@ -215,7 +213,7 @@ class WebsocketHandler {
'mempoolInfo': memPool.getMempoolInfo(),
'vBytesPerSecond': memPool.getVBytesPerSecond(),
'blocks': _blocks,
'conversions': priceUpdater.getLatestPrices(),
'conversions': fiatConversion.getConversionRates(),
'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
'transactions': memPool.getLatestTransactions(),
'backendInfo': backendInfo.getBackendInfo(),
@@ -253,9 +251,9 @@ class WebsocketHandler {
}
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid));
} else {
mempoolBlocks.updateMempoolBlocks(newMempool, true);
mempoolBlocks.updateMempoolBlocks(newMempool);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
@@ -420,57 +418,47 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
if (config.MEMPOOL.AUDIT) {
let projectedBlocks;
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
} else {
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
}
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
await mempoolBlocks.makeBlockTemplates(_memPool);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);
}
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
if (Common.indexingEnabled() && memPool.isInSync()) {
const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
const matchRate = Math.round(score * 100 * 100) / 100;
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
transactions: stripped
}
});
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: censored,
freshTxs: fresh,
matchRate: matchRate,
});
if (block.extras) {
block.extras.matchRate = matchRate;
block.extras.similarity = similarity;
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
transactions: stripped
}
}
} else if (block.extras) {
const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
if (mBlocks?.length && mBlocks[0].transactions) {
block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
});
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: censored,
freshTxs: fresh,
matchRate: matchRate,
});
if (block.extras) {
block.extras.matchRate = matchRate;
}
}
@@ -483,9 +471,9 @@ class WebsocketHandler {
}
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true);
await mempoolBlocks.updateBlockTemplates(_memPool, [], removed);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool, true);
mempoolBlocks.updateMempoolBlocks(_memPool);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();

View File

@@ -19,6 +19,7 @@ interface IConfig {
MEMPOOL_BLOCKS_AMOUNT: number;
INDEXING_BLOCKS_AMOUNT: number;
BLOCKS_SUMMARIES_INDEXING: boolean;
PRICE_FEED_UPDATE_INTERVAL: number;
USE_SECOND_NODE_FOR_MINFEE: boolean;
EXTERNAL_ASSETS: string[];
EXTERNAL_MAX_RETRY: number;
@@ -28,12 +29,10 @@ interface IConfig {
AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
AUDIT: boolean;
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
CPFP_INDEXING: boolean;
MAX_BLOCKS_BULK_QUERY: number;
DISK_CACHE_BLOCK_INTERVAL: number;
RBF_DUAL_NODE: boolean;
};
ESPLORA: {
REST_API_URL: string;
@@ -52,7 +51,6 @@ interface IConfig {
TLS_CERT_PATH: string;
MACAROON_PATH: string;
REST_API_URL: string;
TIMEOUT: number;
};
CLIGHTNING: {
SOCKET: string;
@@ -67,14 +65,12 @@ interface IConfig {
PORT: number;
USERNAME: string;
PASSWORD: string;
TIMEOUT: number;
};
SECOND_CORE_RPC: {
HOST: string;
PORT: number;
USERNAME: string;
PASSWORD: string;
TIMEOUT: number;
};
DATABASE: {
ENABLED: boolean;
@@ -145,6 +141,7 @@ const defaults: IConfig = {
'MEMPOOL_BLOCKS_AMOUNT': 8,
'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
'BLOCKS_SUMMARIES_INDEXING': false,
'PRICE_FEED_UPDATE_INTERVAL': 600,
'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [],
'EXTERNAL_MAX_RETRY': 1,
@@ -152,14 +149,12 @@ const defaults: IConfig = {
'USER_AGENT': 'mempool',
'STDOUT_LOG_MIN_PRIORITY': 'debug',
'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'AUDIT': false,
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'CPFP_INDEXING': false,
'MAX_BLOCKS_BULK_QUERY': 0,
'DISK_CACHE_BLOCK_INTERVAL': 6,
'RBF_DUAL_NODE': false,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',
@@ -173,15 +168,13 @@ const defaults: IConfig = {
'HOST': '127.0.0.1',
'PORT': 8332,
'USERNAME': 'mempool',
'PASSWORD': 'mempool',
'TIMEOUT': 60000,
'PASSWORD': 'mempool'
},
'SECOND_CORE_RPC': {
'HOST': '127.0.0.1',
'PORT': 8332,
'USERNAME': 'mempool',
'PASSWORD': 'mempool',
'TIMEOUT': 60000,
'PASSWORD': 'mempool'
},
'DATABASE': {
'ENABLED': true,
@@ -221,7 +214,6 @@ const defaults: IConfig = {
'TLS_CERT_PATH': '',
'MACAROON_PATH': '',
'REST_API_URL': 'https://localhost:8080',
'TIMEOUT': 10000,
},
'CLIGHTNING': {
'SOCKET': '',

View File

@@ -24,8 +24,7 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
private checkDBFlag() {
if (config.DATABASE.ENABLED === false) {
const stack = new Error().stack;
logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
}
}

View File

@@ -10,12 +10,14 @@ import memPool from './api/mempool';
import diskCache from './api/disk-cache';
import statistics from './api/statistics/statistics';
import websocketHandler from './api/websocket-handler';
import fiatConversion from './api/fiat-conversion';
import bisq from './api/bisq/bisq';
import bisqMarkets from './api/bisq/markets';
import logger from './logger';
import backendInfo from './api/backend-info';
import loadingIndicators from './api/loading-indicators';
import mempool from './api/mempool';
import altMempool from './api/alt-mempool';
import elementsParser from './api/liquid/elements-parser';
import databaseMigration from './api/database-migration';
import syncAssets from './sync-assets';
@@ -35,11 +37,6 @@ import liquidRoutes from './api/liquid/liquid.routes';
import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
import forensicsService from './tasks/lightning/forensics.service';
import priceUpdater from './tasks/price-updater';
import chainTips from './api/chain-tips';
import { AxiosError } from 'axios';
import v8 from 'v8';
import { formatBytes, getBytesUnit } from './utils/format';
class Server {
private wss: WebSocket.Server | undefined;
@@ -47,11 +44,6 @@ class Server {
private app: Application;
private currentBackendRetryInterval = 5;
private maxHeapSize: number = 0;
private heapLogInterval: number = 60;
private warnedHeapCritical: boolean = false;
private lastHeapLogTime: number | null = null;
constructor() {
this.app = express();
@@ -87,18 +79,6 @@ class Server {
async startServer(worker = false): Promise<void> {
logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
if (config.DATABASE.ENABLED) {
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
await databaseMigration.$blocksReindexingTruncate();
}
await databaseMigration.$initializeOrMigrateDatabase();
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
}
}
this.app
.use((req: Request, res: Response, next: NextFunction) => {
res.setHeader('Access-Control-Allow-Origin', '*');
@@ -108,21 +88,34 @@ class Server {
.use(express.text({ type: ['text/plain', 'application/base64'] }))
;
if (config.DATABASE.ENABLED) {
await priceUpdater.$initializeLatestPriceWithDb();
}
this.server = http.createServer(this.app);
this.wss = new WebSocket.Server({ server: this.server });
this.setUpWebsocketHandling();
await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
await syncAssets.syncAssets$();
if (config.MEMPOOL.ENABLED) {
diskCache.loadMempoolCache();
}
if (config.DATABASE.ENABLED) {
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex !== undefined) { // Re-index requests
const tables = process.env.npm_config_reindex.split(',');
logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
await Common.sleep$(5000);
await databaseMigration.$truncateIndexedData(tables);
}
await databaseMigration.$initializeOrMigrateDatabase();
if (Common.indexingEnabled()) {
await indexer.$resetHashratesIndexingState();
}
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
}
}
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
statistics.startStatistics();
}
@@ -135,8 +128,7 @@ class Server {
}
}
priceUpdater.$run();
await chainTips.updateOrphanedBlocks();
fiatConversion.startService();
this.setUpHttpApiRoutes();
@@ -144,8 +136,6 @@ class Server {
this.runMainUpdateLoop();
}
setInterval(() => { this.healthCheck(); }, 2500);
if (config.BISQ.ENABLED) {
bisq.startBisqService();
bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price));
@@ -178,30 +168,25 @@ class Server {
logger.debug(msg);
}
}
await poolsUpdater.updatePoolsJson();
await blocks.$updateBlocks();
await memPool.$updateMempool();
if (config.MEMPOOL.RBF_DUAL_NODE) {
await altMempool.$updateMempool();
}
indexer.$run();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5;
} catch (e: any) {
let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
if (e?.stack) {
loggerMsg += ` Stack trace: ${e.stack}`;
}
// When we get a first Exception, only `logger.debug` it and retry after 5 seconds
// From the second Exception, `logger.warn` the Exception and increase the retry delay
// Maximum retry delay is 60 seconds
} catch (e) {
const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
if (this.currentBackendRetryInterval > 5) {
logger.warn(loggerMsg);
mempool.setOutOfSync();
} else {
logger.debug(loggerMsg);
}
if (e instanceof AxiosError) {
logger.debug(`AxiosError: ${e?.message}`);
}
logger.debug(JSON.stringify(e));
setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
this.currentBackendRetryInterval *= 2;
this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
@@ -212,14 +197,14 @@ class Server {
try {
await fundingTxFetcher.$init();
await networkSyncService.$startService();
await lightningStatsUpdater.$startService();
await forensicsService.$startService();
await lightningStatsUpdater.$startService();
} catch(e) {
logger.err(`Exception in $runLightningBackend. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
await Common.sleep$(1000 * 60);
this.$runLightningBackend();
};
}
}
setUpWebsocketHandling(): void {
if (this.wss) {
@@ -240,7 +225,7 @@ class Server {
memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
}
priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
}
@@ -264,26 +249,6 @@ class Server {
channelsRoutes.initRoutes(this.app);
}
}
healthCheck(): void {
const now = Date.now();
const stats = v8.getHeapStatistics();
this.maxHeapSize = Math.max(stats.used_heap_size, this.maxHeapSize);
const warnThreshold = 0.8 * stats.heap_size_limit;
const byteUnits = getBytesUnit(Math.max(this.maxHeapSize, stats.heap_size_limit));
if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
this.warnedHeapCritical = true;
logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
}
if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);
this.warnedHeapCritical = false;
this.maxHeapSize = 0;
this.lastHeapLogTime = now;
}
}
}
((): Server => new Server())();

View File

@@ -3,71 +3,23 @@ import blocks from './api/blocks';
import mempool from './api/mempool';
import mining from './api/mining/mining';
import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
export interface CoreIndex {
name: string;
synced: boolean;
best_block_height: number;
}
class Indexer {
runIndexer = true;
indexerRunning = false;
tasksRunning: string[] = [];
coreIndexes: CoreIndex[] = [];
/**
* Check which core index is available for indexing
*/
public async checkAvailableCoreIndexes(): Promise<void> {
const updatedCoreIndexes: CoreIndex[] = [];
const indexes: any = await bitcoinClient.getIndexInfo();
for (const indexName in indexes) {
const newState = {
name: indexName,
synced: indexes[indexName].synced,
best_block_height: indexes[indexName].best_block_height,
};
logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);
updatedCoreIndexes.push(newState);
if (indexName === 'coinstatsindex' && newState.synced === true) {
const previousState = this.isCoreIndexReady('coinstatsindex');
// if (!previousState || previousState.synced === false) {
this.runSingleTask('coinStatsIndex');
// }
}
}
this.coreIndexes = updatedCoreIndexes;
}
/**
* Return the best block height if a core index is available, or 0 if not
*
* @param name
* @returns
*/
public isCoreIndexReady(name: string): CoreIndex | null {
for (const index of this.coreIndexes) {
if (index.name === name && index.synced === true) {
return index;
}
}
return null;
}
public reindex(): void {
public reindex() {
if (Common.indexingEnabled()) {
this.runIndexer = true;
}
}
public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
public async runSingleTask(task: 'blocksPrices') {
if (!Common.indexingEnabled()) {
return;
}
@@ -76,27 +28,20 @@ class Indexer {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
setTimeout(() => {
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
this.runSingleTask('blocksPrices');
}, 10000);
} else {
logger.debug(`Blocks prices indexer will run now`, logger.tags.mining);
logger.debug(`Blocks prices indexer will run now`)
await mining.$indexBlockPrices();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
}
}
if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
this.tasksRunning.push(task);
logger.debug(`Indexing coinStatsIndex now`);
await mining.$indexCoinStatsIndex();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
}
}
public async $run(): Promise<void> {
public async $run() {
if (!Common.indexingEnabled() || this.runIndexer === false ||
this.indexerRunning === true || mempool.hasPriority()
) {
@@ -114,15 +59,13 @@ class Indexer {
logger.debug(`Running mining indexer`);
await this.checkAvailableCoreIndexes();
try {
await priceUpdater.$run();
const chainValid = await blocks.$generateBlockDatabase();
if (chainValid === false) {
// Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining);
logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
setTimeout(() => this.reindex(), 10000);
this.indexerRunning = false;
return;
@@ -130,6 +73,7 @@ class Indexer {
this.runSingleTask('blocksPrices');
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();
@@ -148,6 +92,16 @@ class Indexer {
logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
setTimeout(() => this.reindex(), runEvery);
}
async $resetHashratesIndexingState() {
try {
await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new Indexer();

View File

@@ -1,10 +1,8 @@
import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
import { OrphanedBlock } from './api/chain-tips';
import { HeapNode } from './utils/pairing-heap';
import { HeapNode } from "./utils/pairing-heap";
export interface PoolTag {
id: number;
uniqueId: number;
id: number; // mysql row id
name: string;
link: string;
regexes: string; // JSON array
@@ -18,7 +16,6 @@ export interface PoolInfo {
link: string;
blockCount: number;
slug: string;
avgMatchRate: number | null;
}
export interface PoolStats extends PoolInfo {
@@ -66,7 +63,6 @@ interface VinStrippedToScriptsig {
interface VoutStrippedToScriptPubkey {
scriptpubkey_address: string | undefined;
scriptpubkey_asm: string | undefined;
value: number;
}
@@ -148,45 +144,23 @@ export interface TransactionStripped {
}
export interface BlockExtension {
totalFees: number;
medianFee: number; // median fee rate
feeRange: number[]; // fee rate percentiles
reward: number;
matchRate: number | null;
similarity?: number;
pool: {
id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
totalFees?: number;
medianFee?: number;
feeRange?: number[];
reward?: number;
coinbaseTx?: TransactionMinerInfo;
matchRate?: number;
pool?: {
id: number;
name: string;
slug: string;
};
avgFee: number;
avgFeeRate: number;
coinbaseRaw: string;
orphans: OrphanedBlock[] | null;
coinbaseAddress: string | null;
coinbaseSignature: string | null;
coinbaseSignatureAscii: string | null;
virtualSize: number;
avgTxSize: number;
totalInputs: number;
totalOutputs: number;
totalOutputAmt: number;
medianFeeAmt: number | null; // median fee in sats
feePercentiles: number[] | null, // fee percentiles in sats
segwitTotalTxs: number;
segwitTotalSize: number;
segwitTotalWeight: number;
header: string;
utxoSetChange: number;
// Requires coinstatsindex, will be set to NULL otherwise
utxoSetSize: number | null;
totalInputAmt: number | null;
avgFee?: number;
avgFeeRate?: number;
coinbaseRaw?: string;
usd?: number | null;
}
/**
* Note: Everything that is added in here will be automatically returned through
* /api/v1/block and /api/v1/blocks APIs
*/
export interface BlockExtended extends IEsploraApi.Block {
extras: BlockExtension;
}
@@ -294,6 +268,7 @@ interface RequiredParams {
}
export interface ILoadingIndicators { [name: string]: number; }
export interface IConversionRates { [currency: string]: number; }
export interface IBackendInfo {
hostname: string;

View File

@@ -1,7 +1,8 @@
import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
import { BlockExtended, BlockPrice } from '../mempool.interfaces';
import DB from '../database';
import logger from '../logger';
import { Common } from '../api/common';
import { prepareBlock } from '../utils/blocks-utils';
import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2';
@@ -9,90 +10,27 @@ import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
import bitcoinClient from '../api/bitcoin/bitcoin-client';
import config from '../config';
import chainTips from '../api/chain-tips';
import blocks from '../api/blocks';
import BlocksAuditsRepository from './BlocksAuditsRepository';
const BLOCK_DB_FIELDS = `
blocks.hash AS id,
blocks.height,
blocks.version,
UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
blocks.bits,
blocks.nonce,
blocks.difficulty,
blocks.merkle_root,
blocks.tx_count,
blocks.size,
blocks.weight,
blocks.previous_block_hash AS previousblockhash,
UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
blocks.fees AS totalFees,
blocks.median_fee AS medianFee,
blocks.fee_span AS feeRange,
blocks.reward,
pools.unique_id AS poolId,
pools.name AS poolName,
pools.slug AS poolSlug,
blocks.avg_fee AS avgFee,
blocks.avg_fee_rate AS avgFeeRate,
blocks.coinbase_raw AS coinbaseRaw,
blocks.coinbase_address AS coinbaseAddress,
blocks.coinbase_signature AS coinbaseSignature,
blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
blocks.avg_tx_size AS avgTxSize,
blocks.total_inputs AS totalInputs,
blocks.total_outputs AS totalOutputs,
blocks.total_output_amt AS totalOutputAmt,
blocks.median_fee_amt AS medianFeeAmt,
blocks.fee_percentiles AS feePercentiles,
blocks.segwit_total_txs AS segwitTotalTxs,
blocks.segwit_total_size AS segwitTotalSize,
blocks.segwit_total_weight AS segwitTotalWeight,
blocks.header,
blocks.utxoset_change AS utxoSetChange,
blocks.utxoset_size AS utxoSetSize,
blocks.total_input_amt AS totalInputAmts
`;
class BlocksRepository {
/**
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(block: BlockExtended) {
const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate,
median_timestamp, header, coinbase_address,
coinbase_signature, utxoset_size, utxoset_change, avg_tx_size,
total_inputs, total_outputs, total_input_amt, total_output_amt,
fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight,
median_fee_amt, coinbase_signature_ascii
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate
) VALUE (
?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
FROM_UNIXTIME(?), ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?
?, ?, ?, ?
)`;
const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
if (!poolDbId) {
throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
}
const params: any[] = [
block.height,
block.id,
@@ -102,7 +40,7 @@ class BlocksRepository {
block.tx_count,
block.extras.coinbaseRaw,
block.difficulty,
poolDbId.id,
block.extras.pool?.id, // Should always be set to something
block.extras.totalFees,
JSON.stringify(block.extras.feeRange),
block.extras.medianFee,
@@ -114,63 +52,19 @@ class BlocksRepository {
block.previousblockhash,
block.extras.avgFee,
block.extras.avgFeeRate,
block.mediantime,
block.extras.header,
block.extras.coinbaseAddress,
truncatedCoinbaseSignature,
block.extras.utxoSetSize,
block.extras.utxoSetChange,
block.extras.avgTxSize,
block.extras.totalInputs,
block.extras.totalOutputs,
block.extras.totalInputAmt,
block.extras.totalOutputAmt,
block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
block.extras.segwitTotalTxs,
block.extras.segwitTotalSize,
block.extras.segwitTotalWeight,
block.extras.medianFeeAmt,
truncatedCoinbaseSignatureAscii,
];
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
} else {
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
}
/**
* Save newly indexed data from core coinstatsindex
*
* @param utxoSetSize
* @param totalInputAmt
*/
public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
totalInputAmt: number
) : Promise<void> {
try {
const query = `
UPDATE blocks
SET utxoset_size = ?, total_input_amt = ?
WHERE hash = ?
`;
const params: any[] = [
utxoSetSize,
totalInputAmt,
blockHash
];
await DB.query(query, params);
} catch (e: any) {
logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get all block height that have not been indexed between [startHeight, endHeight]
*/
@@ -330,55 +224,6 @@ class BlocksRepository {
}
}
/**
* Get average block health for all blocks for a single pool
*/
public async $getAvgBlockHealthPerPoolId(poolId: number): Promise<number> {
const params: any[] = [];
const query = `
SELECT AVG(blocks_audits.match_rate) AS avg_match_rate
FROM blocks
JOIN blocks_audits ON blocks.height = blocks_audits.height
WHERE blocks.pool_id = ?
`;
params.push(poolId);
try {
const [rows] = await DB.query(query, params);
if (!rows[0] || !rows[0].avg_match_rate) {
return 0;
}
return Math.round(rows[0].avg_match_rate * 100) / 100;
} catch (e) {
logger.err(`Cannot get average block health for pool id ${poolId}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get average block health for all blocks for a single pool
*/
public async $getTotalRewardForPoolId(poolId: number): Promise<number> {
const params: any[] = [];
const query = `
SELECT sum(reward) as total_reward
FROM blocks
WHERE blocks.pool_id = ?
`;
params.push(poolId);
try {
const [rows] = await DB.query(query, params);
if (!rows[0] || !rows[0].total_reward) {
return 0;
}
return rows[0].total_reward;
} catch (e) {
logger.err(`Cannot get total reward for pool id ${poolId}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get the oldest indexed block
*/
@@ -405,17 +250,34 @@ class BlocksRepository {
/**
* Get blocks mined by a specific mining pool
*/
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error('This mining pool does not exist ' + escape(slug));
}
const params: any[] = [];
let query = `
SELECT ${BLOCK_DB_FIELDS}
let query = ` SELECT
blocks.height,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE pool_id = ?`;
params.push(pool.id);
@@ -428,11 +290,11 @@ class BlocksRepository {
LIMIT 10`;
try {
const [rows]: any[] = await DB.query(query, params);
const [rows] = await DB.query(query, params);
const blocks: BlockExtended[] = [];
for (const block of rows) {
blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
for (const block of <object[]>rows) {
blocks.push(prepareBlock(block));
}
return blocks;
@@ -445,21 +307,46 @@ class BlocksRepository {
/**
* Get one block by height
*/
public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
public async $getBlockByHeight(height: number): Promise<object | null> {
try {
const [rows]: any[] = await DB.query(`
SELECT ${BLOCK_DB_FIELDS}
const [rows]: any[] = await DB.query(`SELECT
blocks.height,
hash,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
pools.id as pool_id,
pools.name as pool_name,
pools.link as pool_link,
pools.slug as pool_slug,
pools.addresses as pool_addresses,
pools.regexes as pool_regexes,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE blocks.height = ?`,
[height]
);
WHERE blocks.height = ${height}
`);
if (rows.length <= 0) {
return null;
}
return await this.formatDbBlockIntoExtendedBlock(rows[0]);
rows[0].fee_span = JSON.parse(rows[0].fee_span);
return rows[0];
} catch (e) {
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@@ -472,7 +359,10 @@ class BlocksRepository {
public async $getBlockByHash(hash: string): Promise<object | null> {
try {
const query = `
SELECT ${BLOCK_DB_FIELDS}
SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE hash = ?;
@@ -482,8 +372,9 @@ class BlocksRepository {
if (rows.length <= 0) {
return null;
}
return await this.formatDbBlockIntoExtendedBlock(rows[0]);
rows[0].fee_span = JSON.parse(rows[0].fee_span);
return rows[0];
} catch (e) {
logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@@ -574,15 +465,8 @@ class BlocksRepository {
public async $validateChain(): Promise<boolean> {
try {
const start = new Date().getTime();
const [blocks]: any[] = await DB.query(`
SELECT
height,
hash,
previous_block_hash,
UNIX_TIMESTAMP(blockTimestamp) AS timestamp
FROM blocks
ORDER BY height
`);
const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
let partialMsg = false;
let idx = 1;
@@ -797,7 +681,6 @@ class BlocksRepository {
SELECT height
FROM compact_cpfp_clusters
WHERE height <= ? AND height >= ?
GROUP BY height
ORDER BY height DESC;
`, [currentBlockHeight, minHeight]);
@@ -811,6 +694,7 @@ class BlocksRepository {
logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
return [];
}
/**
@@ -857,7 +741,7 @@ class BlocksRepository {
try {
let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
for (const price of blockPrices) {
query += ` (${price.height}, ${price.priceId}),`;
query += ` (${price.height}, ${price.priceId}),`
}
query = query.slice(0, -1);
await DB.query(query);
@@ -870,132 +754,6 @@ class BlocksRepository {
}
}
}
/**
* Get all indexed blocsk with missing coinstatsindex data
*/
public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
try {
const [blocks] = await DB.query(`
SELECT height, hash
FROM blocks
WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
(utxoset_size IS NULL OR total_input_amt IS NULL)
`);
return blocks;
} catch (e) {
logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Save indexed median fee to avoid recomputing it later
*
* @param id
* @param feePercentiles
*/
public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
try {
await DB.query(`
UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
WHERE hash = ?`,
[JSON.stringify(feePercentiles), feePercentiles[3], id]
);
} catch (e) {
logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Convert a mysql row block into a BlockExtended. Note that you
* must provide the correct field into dbBlk object param
*
* @param dbBlk
*/
private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
const blk: Partial<BlockExtended> = {};
const extras: Partial<BlockExtension> = {};
// IEsploraApi.Block
blk.id = dbBlk.id;
blk.height = dbBlk.height;
blk.version = dbBlk.version;
blk.timestamp = dbBlk.timestamp;
blk.bits = dbBlk.bits;
blk.nonce = dbBlk.nonce;
blk.difficulty = dbBlk.difficulty;
blk.merkle_root = dbBlk.merkle_root;
blk.tx_count = dbBlk.tx_count;
blk.size = dbBlk.size;
blk.weight = dbBlk.weight;
blk.previousblockhash = dbBlk.previousblockhash;
blk.mediantime = dbBlk.mediantime;
// BlockExtension
extras.totalFees = dbBlk.totalFees;
extras.medianFee = dbBlk.medianFee;
extras.feeRange = JSON.parse(dbBlk.feeRange);
extras.reward = dbBlk.reward;
extras.pool = {
id: dbBlk.poolId,
name: dbBlk.poolName,
slug: dbBlk.poolSlug,
};
extras.avgFee = dbBlk.avgFee;
extras.avgFeeRate = dbBlk.avgFeeRate;
extras.coinbaseRaw = dbBlk.coinbaseRaw;
extras.coinbaseAddress = dbBlk.coinbaseAddress;
extras.coinbaseSignature = dbBlk.coinbaseSignature;
extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
extras.avgTxSize = dbBlk.avgTxSize;
extras.totalInputs = dbBlk.totalInputs;
extras.totalOutputs = dbBlk.totalOutputs;
extras.totalOutputAmt = dbBlk.totalOutputAmt;
extras.medianFeeAmt = dbBlk.medianFeeAmt;
extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
extras.segwitTotalSize = dbBlk.segwitTotalSize;
extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
extras.header = dbBlk.header,
extras.utxoSetChange = dbBlk.utxoSetChange;
extras.utxoSetSize = dbBlk.utxoSetSize;
extras.totalInputAmt = dbBlk.totalInputAmt;
extras.virtualSize = dbBlk.weight / 4.0;
// Re-org can happen after indexing so we need to always get the
// latest state from core
extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
// Match rate is not part of the blocks table, but it is part of APIs so we must include it
extras.matchRate = null;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id);
if (auditScore != null) {
extras.matchRate = auditScore.matchRate;
}
}
// If we're missing block summary related field, check if we can populate them on the fly now
if (Common.blocksSummariesIndexingEnabled() &&
(extras.medianFeeAmt === null || extras.feePercentiles === null))
{
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
if (extras.feePercentiles === null) {
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
const summary = blocks.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
}
if (extras.feePercentiles !== null) {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
}
}
export default new BlocksRepository();

View File

@@ -80,48 +80,6 @@ class BlocksSummariesRepository {
logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Get the fee percentiles if the block has already been indexed, [] otherwise
*
* @param id
*/
public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
try {
const [rows]: any[] = await DB.query(`
SELECT transactions
FROM blocks_summaries
WHERE id = ?`,
[id]
);
if (rows === null || rows.length === 0) {
return null;
}
const transactions = JSON.parse(rows[0].transactions);
if (transactions === null) {
return null;
}
transactions.shift(); // Ignore coinbase
transactions.sort((a: any, b: any) => a.fee - b.fee);
const fees = transactions.map((t: any) => t.fee);
return [
fees[0] ?? 0, // min
fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
fees[fees.length - 1] ?? 0, // max
];
} catch (e) {
logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
return null;
}
}
}
export default new BlocksSummariesRepository();

View File

@@ -111,7 +111,7 @@ class CpfpRepository {
}
}
public async $getCluster(clusterRoot: string): Promise<Cluster | void> {
public async $getCluster(clusterRoot: string): Promise<Cluster> {
const [clusterRows]: any = await DB.query(
`
SELECT *
@@ -121,11 +121,8 @@ class CpfpRepository {
[clusterRoot]
);
const cluster = clusterRows[0];
if (cluster?.txs) {
cluster.txs = this.unpack(cluster.txs);
return cluster;
}
return;
cluster.txs = this.unpack(cluster.txs);
return cluster;
}
public async $deleteClustersFrom(height: number): Promise<void> {
@@ -139,9 +136,9 @@ class CpfpRepository {
[height]
) as RowDataPacket[][];
if (rows?.length) {
for (const clusterToDelete of rows) {
const txs = this.unpack(clusterToDelete?.txs);
for (const tx of txs) {
for (let clusterToDelete of rows) {
const txs = this.unpack(clusterToDelete.txs);
for (let tx of txs) {
await transactionRepository.$removeTransaction(tx.txid);
}
}
@@ -207,25 +204,20 @@ class CpfpRepository {
return [];
}
try {
const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
const txs: Ancestor[] = [];
const view = new DataView(arrayBuffer);
for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
const weight = view.getUint32(offset + 32);
const fee = Number(view.getBigUint64(offset + 36));
txs.push({
txid,
weight,
fee
});
}
return txs;
} catch (e) {
logger.warn(`Failed to unpack CPFP cluster. Reason: ` + (e instanceof Error ? e.message : e));
return [];
const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
const txs: Ancestor[] = [];
const view = new DataView(arrayBuffer);
for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
const weight = view.getUint32(offset + 32);
const fee = Number(view.getBigUint64(offset + 36));
txs.push({
txid,
weight,
fee
});
}
return txs;
}
}

View File

@@ -1,4 +1,5 @@
import { Common } from '../api/common';
import config from '../config';
import DB from '../database';
import logger from '../logger';
import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
@@ -20,9 +21,9 @@ class DifficultyAdjustmentsRepository {
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`, logger.tags.mining);
logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
} else {
logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
@@ -54,7 +55,7 @@ class DifficultyAdjustmentsRepository {
const [rows] = await DB.query(query);
return rows as IndexedDifficultyAdjustment[];
} catch (e) {
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -83,7 +84,7 @@ class DifficultyAdjustmentsRepository {
const [rows] = await DB.query(query);
return rows as IndexedDifficultyAdjustment[];
} catch (e) {
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -93,27 +94,27 @@ class DifficultyAdjustmentsRepository {
const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
return rows.map(block => block.height);
} catch (e: any) {
logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
try {
logger.info(`Delete newer difficulty adjustments from height ${height} from the database`, logger.tags.mining);
logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
} catch (e: any) {
logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $deleteLastAdjustment(): Promise<void> {
try {
logger.info(`Delete last difficulty adjustment from the database`, logger.tags.mining);
logger.info(`Delete last difficulty adjustment from the database`);
await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
} catch (e: any) {
logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}

View File

@@ -1,6 +1,5 @@
import { escape } from 'mysql2';
import { Common } from '../api/common';
import mining from '../api/mining/mining';
import DB from '../database';
import logger from '../logger';
import PoolsRepository from './PoolsRepository';
@@ -25,7 +24,7 @@ class HashratesRepository {
try {
await DB.query(query);
} catch (e: any) {
logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -51,7 +50,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -78,7 +77,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -93,7 +92,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows.map(row => row.timestamp);
} catch (e) {
logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -128,7 +127,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows;
} catch (e) {
logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -158,7 +157,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query, [pool.id]);
boundaries = rows[0];
} catch (e) {
logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
}
// Get hashrates entries between boundaries
@@ -173,7 +172,21 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
return rows;
} catch (e) {
logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Set latest run timestamp
*/
public async $setLatestRun(key: string, val: number) {
const query = `UPDATE state SET number = ? WHERE name = ?`;
try {
await DB.query(query, [val, key]);
} catch (e) {
logger.err(`Cannot set last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -192,7 +205,7 @@ class HashratesRepository {
}
return rows[0]['number'];
} catch (e) {
logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
@@ -201,7 +214,7 @@ class HashratesRepository {
* Delete most recent data points for re-indexing
*/
public async $deleteLastEntries() {
logger.info(`Delete latest hashrates data points from the database`, logger.tags.mining);
logger.info(`Delete latest hashrates data points from the database`);
try {
const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
@@ -209,10 +222,10 @@ class HashratesRepository {
await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
}
// Re-run the hashrate indexing to fill up missing data
mining.lastHashrateIndexingDate = null;
mining.lastWeeklyHashrateIndexingDate = null;
await this.$setLatestRun('last_hashrates_indexing', 0);
await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
}
}
@@ -225,10 +238,10 @@ class HashratesRepository {
try {
await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);
// Re-run the hashrate indexing to fill up missing data
mining.lastHashrateIndexingDate = null;
mining.lastWeeklyHashrateIndexingDate = null;
await this.$setLatestRun('last_hashrates_indexing', 0);
await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
}
}
}

View File

@@ -1,5 +1,4 @@
import { Common } from '../api/common';
import poolsParser from '../api/pools-parser';
import config from '../config';
import DB from '../database';
import logger from '../logger';
@@ -10,7 +9,7 @@ class PoolsRepository {
* Get all pools tagging info
*/
public async $getPools(): Promise<PoolTag[]> {
const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
return <PoolTag[]>rows;
}
@@ -18,11 +17,7 @@ class PoolsRepository {
* Get unknown pool tagging info
*/
public async $getUnknownPool(): Promise<PoolTag> {
let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
await poolsParser.$insertUnknownPool();
[rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
}
const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
return <PoolTag>rows[0];
}
@@ -32,25 +27,16 @@ class PoolsRepository {
public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
interval = Common.getSqlInterval(interval);
let query = `
SELECT
COUNT(blocks.height) As blockCount,
pool_id AS poolId,
pools.name AS name,
pools.link AS link,
slug,
AVG(blocks_audits.match_rate) AS avgMatchRate
let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
FROM blocks
JOIN pools on pools.id = pool_id
LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
`;
JOIN pools on pools.id = pool_id`;
if (interval) {
query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY pool_id
ORDER BY COUNT(blocks.height) DESC`;
ORDER BY COUNT(height) DESC`;
try {
const [rows] = await DB.query(query);
@@ -64,7 +50,7 @@ class PoolsRepository {
/**
* Get basic pool info and block count between two timestamp
*/
public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
FROM pools
LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
@@ -80,9 +66,9 @@ class PoolsRepository {
}
/**
* Get a mining pool info
* Get mining pool statistics for one pool
*/
public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
public async $getPool(slug: string): Promise<PoolTag | null> {
const query = `
SELECT *
FROM pools
@@ -95,44 +81,10 @@ class PoolsRepository {
return null;
}
if (parse) {
rows[0].regexes = JSON.parse(rows[0].regexes);
}
if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
} else if (parse) {
rows[0].addresses = JSON.parse(rows[0].addresses);
}
return rows[0];
} catch (e) {
logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get a mining pool info by its unique id
*/
public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
const query = `
SELECT *
FROM pools
WHERE pools.unique_id = ?`;
try {
const [rows]: any[] = await DB.query(query, [id]);
if (rows.length < 1) {
return null;
}
if (parse) {
rows[0].regexes = JSON.parse(rows[0].regexes);
}
rows[0].regexes = JSON.parse(rows[0].regexes);
if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
rows[0].addresses = []; // pools.json only contains mainnet addresses
} else if (parse) {
} else {
rows[0].addresses = JSON.parse(rows[0].addresses);
}
@@ -142,84 +94,6 @@ class PoolsRepository {
throw e;
}
}
/**
* Insert a new mining pool in the database
*
* @param pool
*/
public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
try {
await DB.query(`
INSERT INTO pools
SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`,
[pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id]
);
} catch (e: any) {
logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e));
}
}
/**
* Rename an existing mining pool
*
* @param dbId
* @param newSlug
* @param newName
*/
public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
try {
await DB.query(`
UPDATE pools
SET slug = ?, name = ?
WHERE id = ?`,
[newSlug, newName, dbId]
);
} catch (e: any) {
logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
}
}
/**
* Update an exisiting mining pool link
*
* @param dbId
* @param newLink
*/
public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
try {
await DB.query(`
UPDATE pools
SET link = ?
WHERE id = ?`,
[newLink, dbId]
);
} catch (e: any) {
logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
}
}
/**
* Update an existing mining pool addresses or coinbase tags
*
* @param dbId
* @param addresses
* @param regexes
*/
public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
try {
await DB.query(`
UPDATE pools
SET addresses = ?, regexes = ?
WHERE id = ?`,
[JSON.stringify(addresses), JSON.stringify(regexes), dbId]
);
} catch (e: any) {
logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
}
}
}
export default new PoolsRepository();

View File

@@ -1,228 +1,50 @@
import DB from '../database';
import logger from '../logger';
import priceUpdater from '../tasks/price-updater';
export interface ApiPrice {
time?: number,
USD: number,
EUR: number,
GBP: number,
CAD: number,
CHF: number,
AUD: number,
JPY: number,
}
const ApiPriceFields = `
UNIX_TIMESTAMP(time) as time,
USD,
EUR,
GBP,
CAD,
CHF,
AUD,
JPY
`;
export interface ExchangeRates {
USDEUR: number,
USDGBP: number,
USDCAD: number,
USDCHF: number,
USDAUD: number,
USDJPY: number,
}
export interface Conversion {
prices: ApiPrice[],
exchangeRates: ExchangeRates;
}
export const MAX_PRICES = {
USD: 100000000,
EUR: 100000000,
GBP: 100000000,
CAD: 100000000,
CHF: 100000000,
AUD: 100000000,
JPY: 10000000000,
};
import { Prices } from '../tasks/price-updater';
class PricesRepository {
public async $savePrices(time: number, prices: ApiPrice): Promise<void> {
public async $savePrices(time: number, prices: Prices): Promise<void> {
if (prices.USD === -1) {
// Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues
// As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine
// Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
// As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
return;
}
// Sanity check
for (const currency of Object.keys(prices)) {
if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry"
logger.info(`Ignore BTC${currency} price of ${prices[currency]}`);
prices[currency] = 0;
}
}
try {
await DB.query(`
INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`,
[time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
);
} catch (e) {
} catch (e: any) {
logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getOldestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`
SELECT UNIX_TIMESTAMP(time) AS time
FROM prices
ORDER BY time
LIMIT 1
`);
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getLatestPriceId(): Promise<number | null> {
const [oldestRow] = await DB.query(`
SELECT id
FROM prices
ORDER BY time DESC
LIMIT 1`
);
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`
SELECT UNIX_TIMESTAMP(time) AS time
FROM prices
ORDER BY time DESC
LIMIT 1`
);
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getPricesTimes(): Promise<number[]> {
const [times] = await DB.query(`
SELECT UNIX_TIMESTAMP(time) AS time
FROM prices
WHERE USD != -1
ORDER BY time
`);
if (!Array.isArray(times)) {
return [];
}
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
return times.map(time => time.time);
}
public async $getPricesTimesAndId(): Promise<{time: number, id: number, USD: number}[]> {
const [times] = await DB.query(`
SELECT
UNIX_TIMESTAMP(time) AS time,
id,
USD
FROM prices
ORDER BY time
`);
return times as {time: number, id: number, USD: number}[];
}
public async $getLatestConversionRates(): Promise<ApiPrice> {
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
ORDER BY time DESC
LIMIT 1`
);
if (!Array.isArray(rates) || rates.length === 0) {
return priceUpdater.getEmptyPricesObj();
}
return rates[0] as ApiPrice;
}
public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
try {
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
WHERE UNIX_TIMESTAMP(time) < ?
ORDER BY time DESC
LIMIT 1`,
[timestamp]
);
if (!Array.isArray(rates)) {
throw Error(`Cannot get single historical price from the database`);
}
// Compute fiat exchange rates
let latestPrice = rates[0] as ApiPrice;
if (latestPrice.USD === -1) {
latestPrice = priceUpdater.getEmptyPricesObj();
}
const computeFx = (usd: number, other: number): number =>
Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
const exchangeRates: ExchangeRates = {
USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
};
return {
prices: rates as ApiPrice[],
exchangeRates: exchangeRates
};
} catch (e) {
logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
return null;
}
}
public async $getHistoricalPrices(): Promise<Conversion | null> {
try {
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
ORDER BY time DESC
`);
if (!Array.isArray(rates)) {
throw Error(`Cannot get average historical price from the database`);
}
// Compute fiat exchange rates
let latestPrice = rates[0] as ApiPrice;
if (latestPrice.USD === -1) {
latestPrice = priceUpdater.getEmptyPricesObj();
}
const computeFx = (usd: number, other: number): number =>
Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
const exchangeRates: ExchangeRates = {
USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
};
return {
prices: rates as ApiPrice[],
exchangeRates: exchangeRates
};
} catch (e) {
logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
return null;
}
public async $getPricesTimesAndId(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
return times;
}
}

View File

@@ -3,6 +3,15 @@ import logger from '../logger';
import { Ancestor, CpfpInfo } from '../mempool.interfaces';
import cpfpRepository from './CpfpRepository';
interface CpfpSummary {
txid: string;
cluster: string;
root: string;
txs: Ancestor[];
height: number;
fee_rate: number;
}
class TransactionRepository {
public async $setCluster(txid: string, clusterRoot: string): Promise<void> {
try {
@@ -63,9 +72,7 @@ class TransactionRepository {
const txid = txRows[0].id.toLowerCase();
const clusterId = txRows[0].root.toLowerCase();
const cluster = await cpfpRepository.$getCluster(clusterId);
if (cluster) {
return this.convertCpfp(txid, cluster);
}
return this.convertCpfp(txid, cluster);
}
} catch (e) {
logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
@@ -74,18 +81,13 @@ class TransactionRepository {
}
public async $removeTransaction(txid: string): Promise<void> {
try {
await DB.query(
`
DELETE FROM compact_transactions
WHERE txid = UNHEX(?)
`,
[txid]
);
} catch (e) {
logger.warn('Cannot delete transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
await DB.query(
`
DELETE FROM compact_transactions
WHERE txid = UNHEX(?)
`,
[txid]
);
}
private convertCpfp(txid, cluster): CpfpInfo {
@@ -93,7 +95,7 @@ class TransactionRepository {
const ancestors: Ancestor[] = [];
let matched = false;
for (const tx of (cluster?.txs || [])) {
for (const tx of cluster.txs) {
if (tx.txid === txid) {
matched = true;
} else if (!matched) {

View File

@@ -88,7 +88,5 @@ module.exports = {
verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
walletLock: 'walletlock',
walletPassphrase: 'walletpassphrase',
walletPassphraseChange: 'walletpassphrasechange',
getTxoutSetinfo: 'gettxoutsetinfo',
getIndexInfo: 'getindexinfo',
};
walletPassphraseChange: 'walletpassphrasechange'
}

View File

@@ -72,7 +72,7 @@ class NetworkSyncService {
const graphNodesPubkeys: string[] = [];
for (const node of nodes) {
const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
node.last_update = Math.max(node.last_update ?? 0, latestUpdated);
node.last_update = Math.max(node.last_update, latestUpdated);
await nodesApi.$saveNode(node);
graphNodesPubkeys.push(node.pub_key);
@@ -210,9 +210,6 @@ class NetworkSyncService {
const channels = await channelsApi.$getChannelsWithoutCreatedDate();
for (const channel of channels) {
const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
if (!transaction) {
continue;
}
await DB.query(`
UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
[transaction.timestamp, channel.id]

View File

@@ -22,15 +22,12 @@ class LightningStatsUpdater {
* Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
*/
private async $logStatsDaily(): Promise<void> {
try {
const date = new Date();
Common.setDateMidnight(date);
const networkGraph = await lightningApi.$getNetworkGraph();
await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
logger.debug(`Updated latest network stats`, logger.tags.ln);
} catch (e) {
logger.err(`Exception in $logStatsDaily. Reason: ${(e instanceof Error ? e.message : e)}`);
}
const date = new Date();
Common.setDateMidnight(date);
const networkGraph = await lightningApi.$getNetworkGraph();
await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
logger.debug(`Updated latest network stats`, logger.tags.ln);
}
}

View File

@@ -71,7 +71,7 @@ class FundingTxFetcher {
this.running = false;
}
public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number} | null> {
public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
channelId = Common.channelIntegerIdToShortId(channelId);
if (this.fundingTxCache[channelId]) {
@@ -102,11 +102,6 @@ class FundingTxFetcher {
const rawTx = await bitcoinClient.getRawTransaction(txid);
const tx = await bitcoinClient.decodeRawTransaction(rawTx);
if (!tx || !tx.vout || tx.vout.length < parseInt(outputIdx, 10) + 1 || tx.vout[outputIdx].value === undefined) {
logger.err(`Cannot find blockchain funding tx for channel id ${channelId}. Possible reasons are: bitcoin backend timeout or the channel shortId is not valid`);
return null;
}
this.fundingTxCache[channelId] = {
timestamp: block.time,
txid: txid,

View File

@@ -411,7 +411,7 @@ class LightningStatsImporter {
}
if (totalProcessed > 0) {
logger.info(`Lightning network stats historical import completed`, logger.tags.ln);
logger.notice(`Lightning network stats historical import completed`, logger.tags.ln);
}
} catch (e) {
logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);

View File

@@ -8,18 +8,16 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
import * as https from 'https';
/**
* Maintain the most recent version of pools-v2.json
* Maintain the most recent version of pools.json
*/
class PoolsUpdater {
lastRun: number = 0;
currentSha: string | null = null;
currentSha: string | undefined = undefined;
poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
public async updatePoolsJson(): Promise<void> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
config.MEMPOOL.ENABLED === false
) {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
@@ -33,9 +31,15 @@ class PoolsUpdater {
this.lastRun = now;
if (config.SOCKS5PROXY.ENABLED) {
logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
} else {
logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
}
try {
const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
if (githubSha === null) {
const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
if (githubSha === undefined) {
return;
}
@@ -43,57 +47,32 @@ class PoolsUpdater {
this.currentSha = await this.getShaFromDb();
}
logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
if (this.currentSha !== null && this.currentSha === githubSha) {
logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
if (this.currentSha !== undefined && this.currentSha === githubSha) {
return;
}
// See backend README for more details about the mining pools update process
if (this.currentSha !== null && // If we don't have any mining pool, download it at least once
config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
!process.env.npm_config_update_pools // We're not manually updating mining pool
) {
logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
return;
}
const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
if (this.currentSha === null) {
logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
if (this.currentSha === undefined) {
logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
} else {
logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
}
const poolsJson = await this.query(this.poolsUrl);
if (poolsJson === undefined) {
return;
}
poolsParser.setMiningPools(poolsJson);
if (config.DATABASE.ENABLED === false) { // Don't run db operations
logger.info('Mining pools-v2.json import completed (no database)');
return;
}
try {
await DB.query('START TRANSACTION;');
await poolsParser.migratePoolsJson();
await this.updateDBSha(githubSha);
await DB.query('COMMIT;');
} catch (e) {
logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
await DB.query('ROLLBACK;');
}
logger.info('PoolsUpdater completed');
await poolsParser.migratePoolsJson(poolsJson);
await this.updateDBSha(githubSha);
logger.notice(`PoolsUpdater completed`, logger.tags.mining);
} catch (e) {
this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
logger.err(`PoolsUpdater failed. Will try again in 24h. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
}
}
/**
* Fetch our latest pools-v2.json sha from the db
* Fetch our latest pools.json sha from the db
*/
private async updateDBSha(githubSha: string): Promise<void> {
this.currentSha = githubSha;
@@ -102,46 +81,46 @@ class PoolsUpdater {
await DB.query('DELETE FROM state where name="pools_json_sha"');
await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
} catch (e) {
logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
}
}
}
/**
* Fetch our latest pools-v2.json sha from the db
* Fetch our latest pools.json sha from the db
*/
private async getShaFromDb(): Promise<string | null> {
private async getShaFromDb(): Promise<string | undefined> {
try {
const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
return (rows.length > 0 ? rows[0].string : null);
return (rows.length > 0 ? rows[0].string : undefined);
} catch (e) {
logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
return null;
logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
return undefined;
}
}
/**
* Fetch our latest pools-v2.json sha from github
* Fetch our latest pools.json sha from github
*/
private async fetchPoolsSha(): Promise<string | null> {
private async fetchPoolsSha(): Promise<string | undefined> {
const response = await this.query(this.treeUrl);
if (response !== undefined) {
for (const file of response['tree']) {
if (file['path'] === 'pools-v2.json') {
if (file['path'] === 'pools.json') {
return file['sha'];
}
}
}
logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
return null;
logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`, logger.tags.mining);
return undefined;
}
/**
* Http request wrapper
*/
private async query(path): Promise<any[] | undefined> {
private async query(path): Promise<object | undefined> {
type axiosOptions = {
headers: {
'User-Agent': string

View File

@@ -8,13 +8,12 @@ class BitfinexApi implements PriceFeed {
public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
if (response && response['last_price']) {
return parseInt(response['last_price'], 10);
} else {
return -1;
}
return response ? parseInt(response['last_price'], 10) : -1;
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

View File

@@ -13,11 +13,7 @@ class BitflyerApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
if (response && response['ltp']) {
return parseInt(response['ltp'], 10);
} else {
return -1;
}
return response ? parseInt(response['ltp'], 10) : -1;
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

View File

@@ -13,11 +13,7 @@ class CoinbaseApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
if (response && response['data'] && response['data']['amount']) {
return parseInt(response['data']['amount'], 10);
} else {
return -1;
}
return response ? parseInt(response['data']['amount'], 10) : -1;
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

View File

@@ -13,11 +13,7 @@ class GeminiApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
if (response && response['last']) {
return parseInt(response['last'], 10);
} else {
return -1;
}
return response ? parseInt(response['last'], 10) : -1;
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

View File

@@ -23,14 +23,7 @@ class KrakenApi implements PriceFeed {
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
const ticker = this.getTicker(currency);
if (response && response['result'] && response['result'][ticker] &&
response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0
) {
return parseInt(response['result'][ticker]['c'][0], 10);
} else {
return -1;
}
return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
}
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
@@ -98,7 +91,7 @@ class KrakenApi implements PriceFeed {
}
if (Object.keys(priceHistory).length > 0) {
logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining);
logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining);
}
}
}

View File

@@ -1,8 +1,8 @@
import * as fs from 'fs';
import path from 'path';
import path from "path";
import config from '../config';
import logger from '../logger';
import PricesRepository, { ApiPrice, MAX_PRICES } from '../repositories/PricesRepository';
import PricesRepository from '../repositories/PricesRepository';
import BitfinexApi from './price-feeds/bitfinex-api';
import BitflyerApi from './price-feeds/bitflyer-api';
import CoinbaseApi from './price-feeds/coinbase-api';
@@ -20,18 +20,27 @@ export interface PriceFeed {
}
export interface PriceHistory {
[timestamp: number]: ApiPrice;
[timestamp: number]: Prices;
}
export interface Prices {
USD: number;
EUR: number;
GBP: number;
CAD: number;
CHF: number;
AUD: number;
JPY: number;
}
class PriceUpdater {
public historyInserted = false;
private lastRun = 0;
private lastHistoricalRun = 0;
private running = false;
private feeds: PriceFeed[] = [];
private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
private latestPrices: ApiPrice;
private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
lastRun = 0;
lastHistoricalRun = 0;
running = false;
feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: Prices;
constructor() {
this.latestPrices = this.getEmptyPricesObj();
@@ -43,13 +52,8 @@ class PriceUpdater {
this.feeds.push(new GeminiApi());
}
public getLatestPrices(): ApiPrice {
return this.latestPrices;
}
public getEmptyPricesObj(): ApiPrice {
public getEmptyPricesObj(): Prices {
return {
time: 0,
USD: -1,
EUR: -1,
GBP: -1,
@@ -60,24 +64,7 @@ class PriceUpdater {
};
}
public setRatesChangedCallback(fn: (rates: ApiPrice) => void): void {
this.ratesChangedCallback = fn;
}
/**
* We execute this function before the websocket initialization since
* the websocket init is not done asyncronously
*/
public async $initializeLatestPriceWithDb(): Promise<void> {
this.latestPrices = await PricesRepository.$getLatestConversionRates();
}
public async $run(): Promise<void> {
if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') {
// Coins have no value on testnet/signet, so we want to always show 0
return;
}
if (this.running === true) {
return;
}
@@ -89,11 +76,12 @@ class PriceUpdater {
}
try {
await this.$updatePrice();
if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
await this.$insertHistoricalPrices();
} else {
await this.$updatePrice();
}
} catch (e: any) {
} catch (e) {
logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
}
@@ -124,7 +112,7 @@ class PriceUpdater {
if (feed.currencies.includes(currency)) {
try {
const price = await feed.$fetchPrice(currency);
if (price > -1 && price < MAX_PRICES[currency]) {
if (price > 0) {
prices.push(price);
}
logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
@@ -139,11 +127,7 @@ class PriceUpdater {
// Compute average price, non weighted
prices = prices.filter(price => price > 0);
if (prices.length === 0) {
this.latestPrices[currency] = -1;
} else {
this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
}
this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
}
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
@@ -160,15 +144,7 @@ class PriceUpdater {
}
}
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.latestPrices);
}
this.lastRun = new Date().getTime() / 1000;
if (this.latestPrices.USD === -1) {
this.latestPrices = await PricesRepository.$getLatestConversionRates();
}
}
/**
@@ -237,7 +213,7 @@ class PriceUpdater {
// Group them by timestamp and currency, for example
// grouped[123456789]['USD'] = [1, 2, 3, 4];
const grouped = {};
const grouped: Object = {};
for (const historicalEntry of historicalPrices) {
for (const time in historicalEntry) {
if (existingPriceTimes.includes(parseInt(time, 10))) {
@@ -252,8 +228,8 @@ class PriceUpdater {
for (const currency of this.currencies) {
const price = historicalEntry[time][currency];
if (price > -1 && price < MAX_PRICES[currency]) {
grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
if (price > 0) {
grouped[time][currency].push(parseInt(price, 10));
}
}
}
@@ -262,7 +238,7 @@ class PriceUpdater {
// Average prices and insert everything into the db
let totalInserted = 0;
for (const time in grouped) {
const prices: ApiPrice = this.getEmptyPricesObj();
const prices: Prices = this.getEmptyPricesObj();
for (const currency in grouped[time]) {
if (grouped[time][currency].length === 0) {
continue;

View File

@@ -0,0 +1,33 @@
import { BlockExtended } from '../mempool.interfaces';
export function prepareBlock(block: any): BlockExtended {
return <BlockExtended>{
id: block.id ?? block.hash, // hash for indexed block
timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
height: block.height,
version: block.version,
bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
nonce: block.nonce,
difficulty: block.difficulty,
merkle_root: block.merkle_root ?? block.merkleroot,
tx_count: block.tx_count ?? block.nTx,
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
extras: {
coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
feeRange: block.feeRange ?? block.fee_span,
reward: block.reward ?? block?.extras?.reward,
totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
avgFee: block?.extras?.avgFee ?? block.avg_fee,
avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
pool: block?.extras?.pool ?? (block?.pool_id ? {
id: block.pool_id,
name: block.pool_name,
slug: block.pool_slug,
} : undefined),
usd: block?.extras?.usd ?? block.usd ?? null,
}
};
}

View File

@@ -1,14 +0,0 @@
// simple recursive deep clone for literal-type objects
// does not preserve Dates, Maps, Sets etc
// does not support recursive objects
// properties deeper than maxDepth will be shallow cloned
export function deepClone(obj: any, maxDepth: number = 50, depth: number = 0): any {
let cloned = obj;
if (depth < maxDepth && typeof obj === 'object') {
cloned = Array.isArray(obj) ? [] : {};
for (const key in obj) {
cloned[key] = deepClone(obj[key], maxDepth, depth + 1);
}
}
return cloned;
}

View File

@@ -1,29 +0,0 @@
const byteUnits = ['B', 'kB', 'MB', 'GB', 'TB'];
export function getBytesUnit(bytes: number): string {
if (isNaN(bytes) || !isFinite(bytes)) {
return 'B';
}
let unitIndex = 0;
while (unitIndex < byteUnits.length && bytes > 1024) {
unitIndex++;
bytes /= 1024;
}
return byteUnits[unitIndex];
}
export function formatBytes(bytes: number, toUnit: string, skipUnit = false): string {
if (isNaN(bytes) || !isFinite(bytes)) {
return `${bytes}`;
}
let unitIndex = 0;
while (unitIndex < byteUnits.length && (toUnit && byteUnits[unitIndex] !== toUnit || (!toUnit && bytes > 1024))) {
unitIndex++;
bytes /= 1024;
}
return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`;
}

View File

@@ -1,3 +0,0 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
Signed: AlexLloyd0

View File

@@ -1,3 +0,0 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
Signed: Arooba-git

View File

@@ -1,3 +0,0 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of December 17, 2022.
Signed: piterden

View File

@@ -17,7 +17,7 @@ _Note: address lookups require an Electrum Server and will not work with this co
The default Docker configuration assumes you have the following configuration in your `bitcoin.conf` file:
```ini
```
txindex=1
server=1
rpcuser=mempool
@@ -26,7 +26,7 @@ rpcpassword=mempool
If you want to use different credentials, specify them in the `docker-compose.yml` file:
```yaml
```
api:
environment:
MEMPOOL_BACKEND: "none"
@@ -34,7 +34,6 @@ If you want to use different credentials, specify them in the `docker-compose.ym
CORE_RPC_PORT: "8332"
CORE_RPC_USERNAME: "customuser"
CORE_RPC_PASSWORD: "custompassword"
CORE_RPC_TIMEOUT: "60000"
```
The IP address in the example above refers to Docker's default gateway IP address so that the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
@@ -55,7 +54,7 @@ First, configure `bitcoind` as specified above, and make sure your Electrum Serv
Then, set the following variables in `docker-compose.yml` so Mempool can connect to your Electrum Server:
```yaml
```
api:
environment:
MEMPOOL_BACKEND: "electrum"
@@ -86,7 +85,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
<br/>
`mempool-config.json`:
```json
```
"MEMPOOL": {
"NETWORK": "mainnet",
"BACKEND": "electrum",
@@ -102,23 +101,22 @@ Below we list all settings from `mempool-config.json` and the corresponding over
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"BLOCKS_SUMMARIES_INDEXING": false,
"PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [],
"EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
"STDOUT_LOG_MIN_PRIORITY": "info",
"INDEXING_BLOCKS_AMOUNT": false,
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false,
"MAX_BLOCKS_BULK_QUERY": 0,
"DISK_CACHE_BLOCK_INTERVAL": 6
},
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
MEMPOOL_NETWORK: ""
@@ -134,6 +132,7 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_INITIAL_BLOCKS_AMOUNT: ""
MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: ""
MEMPOOL_BLOCKS_SUMMARIES_INDEXING: ""
MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: ""
MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
MEMPOOL_EXTERNAL_ASSETS: ""
MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
@@ -144,8 +143,6 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_ADVANCED_GBT_AUDIT: ""
MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
MEMPOOL_CPFP_INDEXING: ""
MAX_BLOCKS_BULK_QUERY: ""
DISK_CACHE_BLOCK_INTERVAL: ""
...
```
@@ -156,32 +153,30 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
"CORE_RPC": {
```
"CORE_RPC": {
"HOST": "127.0.0.1",
"PORT": 8332,
"USERNAME": "mempool",
"PASSWORD": "mempool",
"TIMEOUT": 60000
"PASSWORD": "mempool"
},
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
CORE_RPC_HOST: ""
CORE_RPC_PORT: ""
CORE_RPC_USERNAME: ""
CORE_RPC_PASSWORD: ""
CORE_RPC_TIMEOUT: 60000
...
```
<br/>
`mempool-config.json`:
```json
```
"ELECTRUM": {
"HOST": "127.0.0.1",
"PORT": 50002,
@@ -190,7 +185,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
ELECTRUM_HOST: ""
@@ -202,14 +197,14 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"ESPLORA": {
"REST_API_URL": "http://127.0.0.1:3000"
},
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
ESPLORA_REST_API_URL: ""
@@ -219,32 +214,30 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"SECOND_CORE_RPC": {
"HOST": "127.0.0.1",
"PORT": 8332,
"USERNAME": "mempool",
"PASSWORD": "mempool",
"TIMEOUT": 60000
"PASSWORD": "mempool"
},
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
SECOND_CORE_RPC_HOST: ""
SECOND_CORE_RPC_PORT: ""
SECOND_CORE_RPC_USERNAME: ""
SECOND_CORE_RPC_PASSWORD: ""
SECOND_CORE_RPC_TIMEOUT: ""
...
```
<br/>
`mempool-config.json`:
```json
```
"DATABASE": {
"ENABLED": true,
"HOST": "127.0.0.1",
@@ -256,7 +249,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
DATABASE_ENABLED: ""
@@ -271,7 +264,7 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"SYSLOG": {
"ENABLED": true,
"HOST": "127.0.0.1",
@@ -282,7 +275,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
SYSLOG_ENABLED: ""
@@ -296,7 +289,7 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"STATISTICS": {
"ENABLED": true,
"TX_PER_SECOND_SAMPLE_PERIOD": 150
@@ -304,7 +297,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
STATISTICS_ENABLED: ""
@@ -315,7 +308,7 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"BISQ": {
"ENABLED": false,
"DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
@@ -323,7 +316,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
BISQ_ENABLED: ""
@@ -334,7 +327,7 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"SOCKS5PROXY": {
"ENABLED": false,
"HOST": "127.0.0.1",
@@ -345,7 +338,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
SOCKS5PROXY_ENABLED: ""
@@ -359,7 +352,7 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"PRICE_DATA_SERVER": {
"TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
"CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
@@ -367,7 +360,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
PRICE_DATA_SERVER_TOR_URL: ""
@@ -378,7 +371,7 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"LIGHTNING": {
"ENABLED": false
"BACKEND": "lnd"
@@ -390,7 +383,7 @@ Corresponding `docker-compose.yml` overrides:
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
LIGHTNING_ENABLED: false
@@ -405,62 +398,37 @@ Corresponding `docker-compose.yml` overrides:
<br/>
`mempool-config.json`:
```json
```
"LND": {
"TLS_CERT_PATH": ""
"MACAROON_PATH": ""
"REST_API_URL": "https://localhost:8080"
"TIMEOUT": 10000
}
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
LND_TLS_CERT_PATH: ""
LND_MACAROON_PATH: ""
LND_REST_API_URL: "https://localhost:8080"
LND_TIMEOUT: 10000
...
```
<br/>
`mempool-config.json`:
```json
```
"CLIGHTNING": {
"SOCKET": ""
}
```
Corresponding `docker-compose.yml` overrides:
```yaml
```
api:
environment:
CLIGHTNING_SOCKET: ""
...
```
<br/>
`mempool-config.json`:
```json
"MAXMIND": {
"ENABLED": true,
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
"GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
}
```
Corresponding `docker-compose.yml` overrides:
```yaml
api:
environment:
MAXMIND_ENABLED: true,
MAXMIND_GEOLITE2_CITY: "/backend/GeoIP/GeoLite2-City.mmdb",
MAXMIND_GEOLITE2_ASN": "/backend/GeoIP/GeoLite2-ASN.mmdb",
MAXMIND_GEOIP2_ISP": "/backend/GeoIP/GeoIP2-ISP.mmdb"
...
```

View File

@@ -17,7 +17,6 @@ WORKDIR /backend
RUN chown 1000:1000 ./
COPY --from=builder --chown=1000:1000 /build/package ./package/
COPY --from=builder --chown=1000:1000 /build/GeoIP ./GeoIP/
COPY --from=builder --chown=1000:1000 /build/mempool-config.json /build/start.sh /build/wait-for-it.sh ./
USER 1000

View File

@@ -13,6 +13,7 @@
"BLOCK_WEIGHT_UNITS": __MEMPOOL_BLOCK_WEIGHT_UNITS__,
"INITIAL_BLOCKS_AMOUNT": __MEMPOOL_INITIAL_BLOCKS_AMOUNT__,
"MEMPOOL_BLOCKS_AMOUNT": __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__,
"PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
"USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
"EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
"EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__,
@@ -22,19 +23,16 @@
"INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
"BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
"AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
"AUDIT": __MEMPOOL_AUDIT__,
"ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
"ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
"DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
"RBF_DUAL_NODE": __RBF_DUAL_NODE__
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",
"PORT": __CORE_RPC_PORT__,
"USERNAME": "__CORE_RPC_USERNAME__",
"PASSWORD": "__CORE_RPC_PASSWORD__",
"TIMEOUT": __CORE_RPC_TIMEOUT__
"PASSWORD": "__CORE_RPC_PASSWORD__"
},
"ELECTRUM": {
"HOST": "__ELECTRUM_HOST__",
@@ -48,8 +46,7 @@
"HOST": "__SECOND_CORE_RPC_HOST__",
"PORT": __SECOND_CORE_RPC_PORT__,
"USERNAME": "__SECOND_CORE_RPC_USERNAME__",
"PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
"TIMEOUT": __SECOND_CORE_RPC_TIMEOUT__
"PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
},
"DATABASE": {
"ENABLED": __DATABASE_ENABLED__,
@@ -86,8 +83,7 @@
"LND": {
"TLS_CERT_PATH": "__LND_TLS_CERT_PATH__",
"MACAROON_PATH": "__LND_MACAROON_PATH__",
"REST_API_URL": "__LND_REST_API_URL__",
"TIMEOUT": "__LND_TIMEOUT__"
"REST_API_URL": "__LND_REST_API_URL__"
},
"CLIGHTNING": {
"SOCKET": "__CLIGHTNING_SOCKET__"
@@ -111,11 +107,5 @@
"LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
"BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
"BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
},
"MAXMIND": {
"ENABLED": __MAXMIND_ENABLED__,
"GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__",
"GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__",
"GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__"
}
}

View File

@@ -16,28 +16,28 @@ __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__=${MEMPOOL_BLOCKS_SUMMARIES_INDEXING:=false}
__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
__MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
__MEMPOOL_RBF_DUAL_NODE__=${MEMPOOL_RBF_DUAL_NODE:=false}
# CORE_RPC
__CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
__CORE_RPC_PORT__=${CORE_RPC_PORT:=8332}
__CORE_RPC_USERNAME__=${CORE_RPC_USERNAME:=mempool}
__CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool}
__CORE_RPC_TIMEOUT__=${CORE_RPC_TIMEOUT:=60000}
# ELECTRUM
__ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1}
@@ -52,7 +52,6 @@ __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
__SECOND_CORE_RPC_PORT__=${SECOND_CORE_RPC_PORT:=8332}
__SECOND_CORE_RPC_USERNAME__=${SECOND_CORE_RPC_USERNAME:=mempool}
__SECOND_CORE_RPC_PASSWORD__=${SECOND_CORE_RPC_PASSWORD:=mempool}
__SECOND_CORE_RPC_TIMEOUT__=${SECOND_CORE_RPC_TIMEOUT:=60000}
# DATABASE
__DATABASE_ENABLED__=${DATABASE_ENABLED:=true}
@@ -110,18 +109,10 @@ __LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30}
__LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""}
__LND_MACAROON_PATH__=${LND_MACAROON_PATH:=""}
__LND_REST_API_URL__=${LND_REST_API_URL:="https://localhost:8080"}
__LND_TIMEOUT__=${LND_TIMEOUT:=10000}
# CLN
__CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}
# MAXMIND
__MAXMIND_ENABLED__=${MAXMIND_ENABLED:=true}
__MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City.mmdb"}
__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"}
__MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
mkdir -p "${__MEMPOOL_CACHE_DIR__}"
sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@@ -139,27 +130,26 @@ sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}
sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json
sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json
sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json
sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json
sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
sed -i "s/__MEMPOOL_RBF_DUAL_NODE__/${__MEMPOOL_RBF_DUAL_NODE__}/g" mempool-config.json
sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
sed -i "s/__CORE_RPC_USERNAME__/${__CORE_RPC_USERNAME__}/g" mempool-config.json
sed -i "s/__CORE_RPC_PASSWORD__/${__CORE_RPC_PASSWORD__}/g" mempool-config.json
sed -i "s/__CORE_RPC_TIMEOUT__/${__CORE_RPC_TIMEOUT__}/g" mempool-config.json
sed -i "s/__ELECTRUM_HOST__/${__ELECTRUM_HOST__}/g" mempool-config.json
sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json
@@ -171,7 +161,6 @@ sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config
sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json
sed -i "s/__SECOND_CORE_RPC_USERNAME__/${__SECOND_CORE_RPC_USERNAME__}/g" mempool-config.json
sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempool-config.json
sed -i "s/__SECOND_CORE_RPC_TIMEOUT__/${__SECOND_CORE_RPC_TIMEOUT__}/g" mempool-config.json
sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json
sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json
@@ -223,16 +212,8 @@ sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTER
sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json
sed -i "s!__LND_MACAROON_PATH__!${__LND_MACAROON_PATH__}!g" mempool-config.json
sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
sed -i "s!__LND_TIMEOUT__!${__LND_TIMEOUT__}!g" mempool-config.json
# CLN
sed -i "s!__CLIGHTNING_SOCKET__!${__CLIGHTNING_SOCKET__}!g" mempool-config.json
# MAXMIND
sed -i "s!__MAXMIND_ENABLED__!${__MAXMIND_ENABLED__}!g" mempool-config.json
sed -i "s!__MAXMIND_GEOLITE2_CITY__!${__MAXMIND_GEOLITE2_CITY__}!g" mempool-config.json
sed -i "s!__MAXMIND_GEOLITE2_ASN__!${__MAXMIND_GEOLITE2_ASN__}!g" mempool-config.json
sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.json
node /backend/package/index.js

View File

@@ -31,11 +31,9 @@ __LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
__BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets}
__MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
__LIGHTNING__=${LIGHTNING:=false}
__AUDIT__=${AUDIT:=false}
__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
# Export as environment variables to be used by envsubst
export __TESTNET_ENABLED__
@@ -57,11 +55,9 @@ export __LIQUID_WEBSITE_URL__
export __BISQ_WEBSITE_URL__
export __MINING_DASHBOARD__
export __LIGHTNING__
export __AUDIT__
export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
export __HISTORICAL_PRICE__
folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
echo ${folder}

View File

@@ -3,11 +3,6 @@
#backend
cp ./docker/backend/* ./backend/
#geoip-data
mkdir -p ./backend/GeoIP/
wget -O ./backend/GeoIP/GeoLite2-City.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-City.mmdb
wget -O ./backend/GeoIP/GeoLite2-ASN.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-ASN.mmdb
#frontend
localhostIP="127.0.0.1"
cp ./docker/frontend/* ./frontend

2
frontend/.gitignore vendored
View File

@@ -54,8 +54,6 @@ src/resources/assets-testnet.json
src/resources/assets-testnet.minimal.json
src/resources/pools.json
src/resources/mining-pools/*
src/resources/**/*.mp4
src/resources/**/*.vtt
# environment config
mempool-frontend-config.json

View File

@@ -1,9 +1,7 @@
[main]
host = https://www.transifex.com
[o:mempool:p:mempool:r:frontend-src-locale-messages-xlf--master]
[mempool.frontend-src-locale-messages-xlf--master]
file_filter = frontend/src/locale/messages.<lang>.xlf
source_file = frontend/src/locale/messages.en-US.xlf
source_lang = en-US
type = XLIFF
type = XLIFF

View File

@@ -111,7 +111,7 @@ https://www.transifex.com/mempool/mempool/dashboard/
* Spanish @maxhodler @bisqes
* Persian @techmix
* French @Bayernatoor
* Korean @kcalvinalvinn @sogoagain
* Korean @kcalvinalvinn
* Italian @HodlBits
* Hebrew @rapidlab309
* Georgian @wyd_idk
@@ -132,4 +132,3 @@ https://www.transifex.com/mempool/mempool/dashboard/
* Russian @TonyCrusoe @Bitconan
* Romanian @mirceavesa
* Macedonian @SkechBoy
* Nepalese @kebinm

View File

@@ -38,10 +38,6 @@
"translation": "src/locale/messages.de.xlf",
"baseHref": "/de/"
},
"da": {
"translation": "src/locale/messages.da.xlf",
"baseHref": "/da/"
},
"es": {
"translation": "src/locale/messages.es.xlf",
"baseHref": "/es/"
@@ -142,10 +138,6 @@
"translation": "src/locale/messages.hi.xlf",
"baseHref": "/hi/"
},
"ne": {
"translation": "src/locale/messages.ne.xlf",
"baseHref": "/ne/"
},
"lt": {
"translation": "src/locale/messages.lt.xlf",
"baseHref": "/lt/"

View File

@@ -1,4 +1,4 @@
import { defineConfig } from 'cypress';
import { defineConfig } from 'cypress'
export default defineConfig({
projectId: 'ry4br7',
@@ -12,18 +12,12 @@ export default defineConfig({
},
chromeWebSecurity: false,
e2e: {
setupNodeEvents(on: any, config: any) {
const fs = require('fs');
const CONFIG_FILE = 'mempool-frontend-config.json';
if (fs.existsSync(CONFIG_FILE)) {
let contents = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
config.env.BASE_MODULE = contents.BASE_MODULE ? contents.BASE_MODULE : 'mempool';
} else {
config.env.BASE_MODULE = 'mempool';
}
return config;
// We've imported your old cypress plugins here.
// You may want to clean this up later by importing these.
setupNodeEvents(on, config) {
return require('./cypress/plugins/index.js')(on, config)
},
baseUrl: 'http://localhost:4200',
specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
},
});
})

View File

@@ -1,5 +1,5 @@
describe('Bisq', () => {
const baseModule = Cypress.env('BASE_MODULE');
const baseModule = Cypress.env("BASE_MODULE");
const basePath = '';
beforeEach(() => {
@@ -20,7 +20,7 @@ describe('Bisq', () => {
cy.waitForSkeletonGone();
});
describe('transactions', () => {
describe("transactions", () => {
it('loads the transactions screen', () => {
cy.visit(`${basePath}`);
cy.waitForSkeletonGone();
@@ -30,9 +30,9 @@ describe('Bisq', () => {
});
const filters = [
'Asset listing fee', 'Blind vote', 'Compensation request',
'Genesis', 'Irregular', 'Lockup', 'Pay trade fee', 'Proof of burn',
'Proposal', 'Reimbursement request', 'Transfer BSQ', 'Unlock', 'Vote reveal'
"Asset listing fee", "Blind vote", "Compensation request",
"Genesis", "Irregular", "Lockup", "Pay trade fee", "Proof of burn",
"Proposal", "Reimbursement request", "Transfer BSQ", "Unlock", "Vote reveal"
];
filters.forEach((filter) => {
it.only(`filters the transaction screen by ${filter}`, () => {
@@ -49,7 +49,7 @@ describe('Bisq', () => {
});
});
it('filters using multiple criteria', () => {
it("filters using multiple criteria", () => {
const filters = ['Proposal', 'Lockup', 'Unlock'];
cy.visit(`${basePath}/transactions`);
cy.waitForSkeletonGone();

View File

@@ -1,5 +1,5 @@
describe('Liquid', () => {
const baseModule = Cypress.env('BASE_MODULE');
const baseModule = Cypress.env("BASE_MODULE");
const basePath = '';
beforeEach(() => {
@@ -158,10 +158,10 @@ describe('Liquid', () => {
it('show empty unblinded TX', () => {
cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
});
@@ -169,8 +169,8 @@ describe('Liquid', () => {
it('show invalid unblinded TX hex', () => {
cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=123`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vin tr').should('have.class', '');
cy.get('.table-tx-vout tr').should('have.class', '');
cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
});

View File

@@ -1,5 +1,5 @@
describe('Liquid Testnet', () => {
const baseModule = Cypress.env('BASE_MODULE');
const baseModule = Cypress.env("BASE_MODULE");
const basePath = '/testnet';
beforeEach(() => {
@@ -109,10 +109,10 @@ describe('Liquid Testnet', () => {
it('show empty unblinded TX', () => {
cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
});
@@ -120,8 +120,8 @@ describe('Liquid Testnet', () => {
it('show invalid unblinded TX hex', () => {
cy.visit(`${basePath}/tx/2477f220eef1d03f8ffa4a2861c275d155c3562adf0d79523aeeb0c59ee611ba#blinded=5000`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
cy.get('.table-tx-vin tr').should('have.class', '');
cy.get('.table-tx-vout tr').should('have.class', '');
cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
});

View File

@@ -1,6 +1,6 @@
import { emitMempoolInfo, dropWebSocket } from '../../support/websocket';
import { emitMempoolInfo, dropWebSocket } from "../../support/websocket";
const baseModule = Cypress.env('BASE_MODULE');
const baseModule = Cypress.env("BASE_MODULE");
//Credit: https://github.com/bahmutov/cypress-examples/blob/6cedb17f83a3bb03ded13cf1d6a3f0656ca2cdf5/docs/recipes/overlapping-elements.md
@@ -64,7 +64,7 @@ describe('Mainnet', () => {
it('loads the status screen', () => {
cy.visit('/status');
cy.get('#mempool-block-0').should('be.visible');
cy.get('[id^="bitcoin-block-"]').should('have.length', 22);
cy.get('[id^="bitcoin-block-"]').should('have.length', 8);
cy.get('.footer').should('be.visible');
cy.get('.row > :nth-child(1)').invoke('text').then((text) => {
expect(text).to.match(/Incoming transactions.* vB\/s/);
@@ -219,11 +219,11 @@ describe('Mainnet', () => {
describe('blocks navigation', () => {
describe('keyboard events', () => {
it('loads first blockchain block visible and keypress arrow right', () => {
it('loads first blockchain blocks visible and keypress arrow right', () => {
cy.viewport('macbook-16');
cy.visit('/');
cy.waitForSkeletonGone();
cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
cy.get('.blockchain-blocks-0 > a').click().then(() => {
cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
cy.waitForPageIdle();
@@ -233,11 +233,11 @@ describe('Mainnet', () => {
});
});
it('loads first blockchain block visible and keypress arrow left', () => {
it('loads first blockchain blocks visible and keypress arrow left', () => {
cy.viewport('macbook-16');
cy.visit('/');
cy.waitForSkeletonGone();
cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
cy.get('.blockchain-blocks-0 > a').click().then(() => {
cy.waitForPageIdle();
cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
@@ -246,11 +246,11 @@ describe('Mainnet', () => {
});
});
it.skip('loads last blockchain block and keypress arrow right', () => { //Skip for now as "last" doesn't really work with infinite scrolling
it('loads last blockchain blocks and keypress arrow right', () => {
cy.viewport('macbook-16');
cy.visit('/');
cy.waitForSkeletonGone();
cy.get('bitcoin-block-offset-0-index-7').click().then(() => {
cy.get('.blockchain-blocks-4 > a').click().then(() => {
cy.waitForPageIdle();
// block 6
@@ -309,7 +309,7 @@ describe('Mainnet', () => {
cy.viewport('macbook-16');
cy.visit('/');
cy.waitForSkeletonGone();
cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
cy.get('.blockchain-blocks-0 > a').click().then(() => {
cy.waitForPageIdle();
cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
@@ -339,14 +339,14 @@ describe('Mainnet', () => {
cy.visit('/');
cy.waitForSkeletonGone();
cy.changeNetwork('testnet');
cy.changeNetwork('signet');
cy.changeNetwork('mainnet');
cy.changeNetwork("testnet");
cy.changeNetwork("signet");
cy.changeNetwork("mainnet");
});
it.skip('loads the dashboard with the skeleton blocks', () => {
cy.mockMempoolSocket();
cy.visit('/');
cy.visit("/");
cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');

View File

@@ -1,4 +1,4 @@
const baseModule = Cypress.env('BASE_MODULE');
const baseModule = Cypress.env("BASE_MODULE");
describe('Mainnet - Mining Features', () => {
beforeEach(() => {

View File

@@ -1,6 +1,6 @@
import { emitMempoolInfo } from '../../support/websocket';
import { emitMempoolInfo } from "../../support/websocket";
const baseModule = Cypress.env('BASE_MODULE');
const baseModule = Cypress.env("BASE_MODULE");
describe('Signet', () => {
beforeEach(() => {
@@ -25,7 +25,7 @@ describe('Signet', () => {
it.skip('loads the dashboard with the skeleton blocks', () => {
cy.mockMempoolSocket();
cy.visit('/signet');
cy.visit("/signet");
cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');
@@ -35,7 +35,7 @@ describe('Signet', () => {
emitMempoolInfo({
'params': {
'network': 'signet'
"network": "signet"
}
});

View File

@@ -1,6 +1,6 @@
import { emitMempoolInfo } from '../../support/websocket';
import { confirmAddress, emitMempoolInfo, sendWsMock, showNewTx, startTrackingAddress } from "../../support/websocket";
const baseModule = Cypress.env('BASE_MODULE');
const baseModule = Cypress.env("BASE_MODULE");
describe('Testnet', () => {
beforeEach(() => {
@@ -25,7 +25,7 @@ describe('Testnet', () => {
it.skip('loads the dashboard with the skeleton blocks', () => {
cy.mockMempoolSocket();
cy.visit('/testnet');
cy.visit("/testnet");
cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');

View File

@@ -0,0 +1,13 @@
const fs = require('fs');
const CONFIG_FILE = 'mempool-frontend-config.json';
module.exports = (on, config) => {
if (fs.existsSync(CONFIG_FILE)) {
let contents = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
config.env.BASE_MODULE = contents.BASE_MODULE ? contents.BASE_MODULE : 'mempool';
} else {
config.env.BASE_MODULE = 'mempool';
}
return config;
}

View File

@@ -17,10 +17,10 @@
"LIQUID_WEBSITE_URL": "https://liquid.network",
"BISQ_WEBSITE_URL": "https://bisq.markets",
"MINING_DASHBOARD": true,
"AUDIT": false,
"MAINNET_BLOCK_AUDIT_START_HEIGHT": 0,
"TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
"SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
"LIGHTNING": false,
"HISTORICAL_PRICE": true
"FULL_RBF_ENABLED": false,
"ALT_BACKEND_ENABLED": false
}

View File

@@ -58,7 +58,7 @@
},
"optionalDependencies": {
"@cypress/schematic": "^2.4.0",
"cypress": "^12.7.0",
"cypress": "^12.3.0",
"cypress-fail-on-console-error": "~4.0.2",
"cypress-wait-until": "^1.7.2",
"mock-socket": "~9.1.5",
@@ -7010,9 +7010,9 @@
"peer": true
},
"node_modules/cypress": {
"version": "12.7.0",
"resolved": "https://registry.npmjs.org/cypress/-/cypress-12.7.0.tgz",
"integrity": "sha512-7rq+nmhzz0u6yabCFyPtADU2OOrYt6pvUau9qV7xyifJ/hnsaw/vkr0tnLlcuuQKUAOC1v1M1e4Z0zG7S0IAvA==",
"version": "12.3.0",
"resolved": "https://registry.npmjs.org/cypress/-/cypress-12.3.0.tgz",
"integrity": "sha512-ZQNebibi6NBt51TRxRMYKeFvIiQZ01t50HSy7z/JMgRVqBUey3cdjog5MYEbzG6Ktti5ckDt1tfcC47lmFwXkw==",
"hasInstallScript": true,
"optional": true,
"dependencies": {
@@ -7033,7 +7033,7 @@
"commander": "^5.1.0",
"common-tags": "^1.8.0",
"dayjs": "^1.10.4",
"debug": "^4.3.4",
"debug": "^4.3.2",
"enquirer": "^2.3.6",
"eventemitter2": "6.4.7",
"execa": "4.1.0",
@@ -7159,23 +7159,6 @@
"node": ">= 6"
}
},
"node_modules/cypress/node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"optional": true,
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/cypress/node_modules/execa": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",
@@ -22293,9 +22276,9 @@
"peer": true
},
"cypress": {
"version": "12.7.0",
"resolved": "https://registry.npmjs.org/cypress/-/cypress-12.7.0.tgz",
"integrity": "sha512-7rq+nmhzz0u6yabCFyPtADU2OOrYt6pvUau9qV7xyifJ/hnsaw/vkr0tnLlcuuQKUAOC1v1M1e4Z0zG7S0IAvA==",
"version": "12.3.0",
"resolved": "https://registry.npmjs.org/cypress/-/cypress-12.3.0.tgz",
"integrity": "sha512-ZQNebibi6NBt51TRxRMYKeFvIiQZ01t50HSy7z/JMgRVqBUey3cdjog5MYEbzG6Ktti5ckDt1tfcC47lmFwXkw==",
"optional": true,
"requires": {
"@cypress/request": "^2.88.10",
@@ -22315,7 +22298,7 @@
"commander": "^5.1.0",
"common-tags": "^1.8.0",
"dayjs": "^1.10.4",
"debug": "^4.3.4",
"debug": "^4.3.2",
"enquirer": "^2.3.6",
"eventemitter2": "6.4.7",
"execa": "4.1.0",
@@ -22399,15 +22382,6 @@
"integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==",
"optional": true
},
"debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"optional": true,
"requires": {
"ms": "2.1.2"
}
},
"execa": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",

View File

@@ -110,7 +110,7 @@
},
"optionalDependencies": {
"@cypress/schematic": "^2.4.0",
"cypress": "^12.7.0",
"cypress": "^12.3.0",
"cypress-fail-on-console-error": "~4.0.2",
"cypress-wait-until": "^1.7.2",
"mock-socket": "~9.1.5",
@@ -119,4 +119,4 @@
"scarfSettings": {
"enabled": false
}
}
}

View File

@@ -3,8 +3,8 @@ const fs = require('fs');
let PROXY_CONFIG = require('./proxy.conf');
PROXY_CONFIG.forEach(entry => {
entry.target = entry.target.replace("mempool.space", "mempool-staging.fra.mempool.space");
entry.target = entry.target.replace("liquid.network", "liquid-staging.fra.mempool.space");
entry.target = entry.target.replace("mempool.space", "mempool-staging.tk7.mempool.space");
entry.target = entry.target.replace("liquid.network", "liquid-staging.tk7.mempool.space");
entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space");
});

View File

@@ -72,10 +72,22 @@ export const chartColors = [
];
export const poolsColor = {
'unknown': '#FDD835',
};
'foundryusa': '#D81B60',
'antpool': '#8E24AA',
'f2pool': '#5E35B1',
'poolin': '#3949AB',
'binancepool': '#1E88E5',
'viabtc': '#039BE5',
'btccom': '#00897B',
'braiinspool': '#00ACC1',
'sbicrypto': '#43A047',
'marapool': '#7CB342',
'luxor': '#C0CA33',
'unknown': '#FDD835',
'okkong': '#FFB300',
}
export const feeLevels = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
export const feeLevels = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
export interface Language {
@@ -87,9 +99,9 @@ export const languages: Language[] = [
{ code: 'ar', name: 'العربية' }, // Arabic
// { code: 'bg', name: 'Български' }, // Bulgarian
// { code: 'bs', name: 'Bosanski' }, // Bosnian
// { code: 'ca', name: 'Català' }, // Catalan
{ code: 'ca', name: 'Català' }, // Catalan
{ code: 'cs', name: 'Čeština' }, // Czech
{ code: 'da', name: 'Dansk' }, // Danish
// { code: 'da', name: 'Dansk' }, // Danish
{ code: 'de', name: 'Deutsch' }, // German
// { code: 'et', name: 'Eesti' }, // Estonian
// { code: 'el', name: 'Ελληνικά' }, // Greek
@@ -104,7 +116,6 @@ export const languages: Language[] = [
// { code: 'hr', name: 'Hrvatski' }, // Croatian
// { code: 'id', name: 'Bahasa Indonesia' },// Indonesian
{ code: 'hi', name: 'हिन्दी' }, // Hindi
{ code: 'ne', name: 'नेपाली' }, // Nepalese
{ code: 'it', name: 'Italiano' }, // Italian
{ code: 'he', name: 'עברית' }, // Hebrew
{ code: 'ka', name: 'ქართული' }, // Georgian
@@ -136,127 +147,12 @@ export const languages: Language[] = [
];
export const specialBlocks = {
'0': {
labelEvent: 'Genesis',
labelEventCompleted: 'The Genesis of Bitcoin',
networks: ['mainnet', 'testnet'],
},
'210000': {
labelEvent: 'Bitcoin\'s 1st Halving',
labelEventCompleted: 'Block Subsidy has halved to 25 BTC per block',
networks: ['mainnet', 'testnet'],
},
'420000': {
labelEvent: 'Bitcoin\'s 2nd Halving',
labelEventCompleted: 'Block Subsidy has halved to 12.5 BTC per block',
networks: ['mainnet', 'testnet'],
},
'630000': {
labelEvent: 'Bitcoin\'s 3rd Halving',
labelEventCompleted: 'Block Subsidy has halved to 6.25 BTC per block',
networks: ['mainnet', 'testnet'],
},
'709632': {
labelEvent: 'Taproot 🌱 activation',
labelEventCompleted: 'Taproot 🌱 has been activated!',
networks: ['mainnet'],
},
'840000': {
labelEvent: 'Bitcoin\'s 4th Halving',
labelEvent: 'Halving 🥳',
labelEventCompleted: 'Block Subsidy has halved to 3.125 BTC per block',
networks: ['mainnet', 'testnet'],
},
'1050000': {
labelEvent: 'Bitcoin\'s 5th Halving',
labelEventCompleted: 'Block Subsidy has halved to 1.5625 BTC per block',
networks: ['mainnet', 'testnet'],
},
'1260000': {
labelEvent: 'Bitcoin\'s 6th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.78125 BTC per block',
networks: ['mainnet', 'testnet'],
},
'1470000': {
labelEvent: 'Bitcoin\'s 7th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.390625 BTC per block',
networks: ['mainnet', 'testnet'],
},
'1680000': {
labelEvent: 'Bitcoin\'s 8th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.1953125 BTC per block',
networks: ['mainnet', 'testnet'],
},
'1890000': {
labelEvent: 'Bitcoin\'s 9th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.09765625 BTC per block',
networks: ['mainnet', 'testnet'],
},
'2100000': {
labelEvent: 'Bitcoin\'s 10th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.04882812 BTC per block',
networks: ['mainnet', 'testnet'],
},
'2310000': {
labelEvent: 'Bitcoin\'s 11th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.02441406 BTC per block',
networks: ['mainnet', 'testnet'],
},
'2520000': {
labelEvent: 'Bitcoin\'s 12th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.01220703 BTC per block',
networks: ['mainnet', 'testnet'],
},
'2730000': {
labelEvent: 'Bitcoin\'s 13th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.00610351 BTC per block',
networks: ['mainnet', 'testnet'],
},
'2940000': {
labelEvent: 'Bitcoin\'s 14th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.00305175 BTC per block',
networks: ['mainnet', 'testnet'],
},
'3150000': {
labelEvent: 'Bitcoin\'s 15th Halving',
labelEventCompleted: 'Block Subsidy has halved to 0.00152587 BTC per block',
networks: ['mainnet', 'testnet'],
}
};
export const fiatCurrencies = {
AUD: {
name: 'Australian Dollar',
code: 'AUD',
indexed: true,
},
CAD: {
name: 'Canadian Dollar',
code: 'CAD',
indexed: true,
},
CHF: {
name: 'Swiss Franc',
code: 'CHF',
indexed: true,
},
EUR: {
name: 'Euro',
code: 'EUR',
indexed: true,
},
GBP: {
name: 'Pound Sterling',
code: 'GBP',
indexed: true,
},
JPY: {
name: 'Japanese Yen',
code: 'JPY',
indexed: true,
},
USD: {
name: 'US Dollar',
code: 'USD',
indexed: true,
},
};

Some files were not shown because too many files have changed in this diff Show More