machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/clightning-wrong-channel-id

Browse Source
This commit is contained in:
wiz
2022-08-11 18:38:11 +09:00
committed by GitHub
parent dc7231537f 8ed1644081
commit c85b6d53c5
29 changed files with 511 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/src/api/blocks.ts
View File

@@ -22,6 +22,8 @@ import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -457,6 +459,19 @@ class Blocks {
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
await blocksRepository.$saveBlockPrices([{
height: blockExtended.height,
priceId: lastestPriceId,
}]);
} else {
logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
setTimeout(() => {
indexer.runSingleTask('blocksPrices');
}, 10000);
}
// Save blocks summary for visualization if it's enabled
if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true);

30
backend/src/api/explorer/channels.api.ts
View File

@@ -96,7 +96,31 @@ class ChannelsApi {
public async $getChannel(id: string): Promise<any> {
try {
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`;
const query = `
SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
channels.*,
ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
WHERE (
ns1.id = (
SELECT MAX(id)
FROM node_stats
WHERE public_key = channels.node1_public_key
)
AND ns2.id = (
SELECT MAX(id)
FROM node_stats
WHERE public_key = channels.node2_public_key
)
)
AND channels.id = ?
`;
const [rows]: any = await DB.query(query, [id]);
if (rows[0]) {
return this.convertChannel(rows[0]);
@@ -289,6 +313,8 @@ class ChannelsApi {
'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
'updated_at': channel.node1_updated_at,
'longitude': channel.node1_longitude,
'latitude': channel.node1_latitude,
},
'node_right': {
'alias': channel.alias_right,
@@ -302,6 +328,8 @@ class ChannelsApi {
'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
'updated_at': channel.node2_updated_at,
'longitude': channel.node2_longitude,
'latitude': channel.node2_latitude,
},
};
}

3
backend/src/api/explorer/channels.routes.ts
View File

@@ -32,6 +32,9 @@ class ChannelsRoutes {
res.status(404).send('Channel not found');
return;
}
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channel);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);

2
backend/src/api/explorer/nodes.api.ts
View File

@@ -168,7 +168,7 @@ class NodesApi {
}
}
public async $getNodesISP(groupBy: string, showTor: boolean) {
public async $getNodesISPRanking(groupBy: string, showTor: boolean) {
try {
const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`;

2
backend/src/api/explorer/nodes.routes.ts
View File

@@ -79,7 +79,7 @@ class NodesRoutes {
return;
}
const nodesPerAs = await nodesApi.$getNodesISP(groupBy, showTor);
const nodesPerAs = await nodesApi.$getNodesISPRanking(groupBy, showTor);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');

14
backend/src/api/mining/mining.ts
View File

@@ -473,7 +473,7 @@ class Mining {
for (const block of blocksWithoutPrices) {
// Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
if (block.height < 68951) {
if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
blocksPrices.push({
height: block.height,
priceId: prices[0].id,
@@ -492,11 +492,11 @@ class Mining {
if (blocksPrices.length >= 100000) {
totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0;
}
@@ -504,11 +504,11 @@ class Mining {
if (blocksPrices.length > 0) {
totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
}
} catch (e) {

4
backend/src/config.ts
View File

@@ -24,6 +24,8 @@ interface IConfig {
USER_AGENT: string;
STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
};
ESPLORA: {
REST_API_URL: string;
@@ -136,6 +138,8 @@ const defaults: IConfig = {
'USER_AGENT': 'mempool',
'STDOUT_LOG_MIN_PRIORITY': 'debug',
'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

29
backend/src/indexer.ts
View File

@@ -6,13 +6,12 @@ import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
class Indexer {
runIndexer = true;
indexerRunning = false;
constructor() {
}
tasksRunning: string[] = [];
public reindex() {
if (Common.indexingEnabled()) {
@@ -20,6 +19,28 @@ class Indexer {
}
}
public async runSingleTask(task: 'blocksPrices') {
if (!Common.indexingEnabled()) {
return;
}
if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
setTimeout(() => {
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
this.runSingleTask('blocksPrices');
}, 10000);
} else {
logger.debug(`Blocks prices indexer will run now`)
await mining.$indexBlockPrices();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
}
}
}
public async $run() {
if (!Common.indexingEnabled() || this.runIndexer === false ||
this.indexerRunning === true || mempool.hasPriority()
@@ -50,7 +71,7 @@ class Indexer {
return;
}
await mining.$indexBlockPrices();
this.runSingleTask('blocksPrices');
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory();

5
backend/src/repositories/PricesRepository.ts
View File

@@ -27,6 +27,11 @@ class PricesRepository {
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getLatestPriceId(): Promise<number | null> {
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;

13
backend/src/tasks/pools-updater.ts
View File

@@ -12,14 +12,11 @@ import * as https from 'https';
*/
class PoolsUpdater {
lastRun: number = 0;
currentSha: any = undefined;
poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
currentSha: string | undefined = undefined;
poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
constructor() {
}
public async updatePoolsJson() {
public async updatePoolsJson(): Promise<void> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
@@ -77,7 +74,7 @@ class PoolsUpdater {
/**
* Fetch our latest pools.json sha from the db
*/
private async updateDBSha(githubSha: string) {
private async updateDBSha(githubSha: string): Promise<void> {
this.currentSha = githubSha;
if (config.DATABASE.ENABLED === true) {
try {

9
backend/src/tasks/price-updater.ts
View File

@@ -1,4 +1,5 @@
import * as fs from 'fs';
import { Common } from '../api/common';
import config from '../config';
import logger from '../logger';
import PricesRepository from '../repositories/PricesRepository';
@@ -34,10 +35,10 @@ export interface Prices {
}
class PriceUpdater {
historyInserted: boolean = false;
lastRun: number = 0;
lastHistoricalRun: number = 0;
running: boolean = false;
public historyInserted = false;
lastRun = 0;
lastHistoricalRun = 0;
running = false;
feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: Prices;