machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into fix_electrum_api

Browse Source
This commit is contained in:
Felipe Knorr Kuhn
2022-07-06 13:36:36 -07:00
committed by GitHub
parent 40957158db ec2d079da4
commit af2548d705
14 changed files with 126 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 22;
private static currentVersion = 23;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -231,6 +231,18 @@ class DatabaseMigration {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
}
if (databaseSchemaVersion < 23) {
await this.$executeQuery('TRUNCATE `prices`');
await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
}
} catch (e) {
throw e;
}

19
backend/src/api/mining.ts
View File

@@ -9,6 +9,7 @@ import loadingIndicators from './loading-indicators';
import { escape } from 'mysql2';
import indexer from '../indexer';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import config from '../config';
class Mining {
constructor() {
@@ -304,7 +305,7 @@ class Mining {
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 86400000;
// Skip already indexed weeks
// Skip already indexed days
if (indexedTimestamp.includes(toTimestamp / 1000)) {
toTimestamp -= 86400000;
++totalIndexed;
@@ -315,7 +316,7 @@ class Mining {
// we are currently indexing has complete data)
const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
if (blockStatsPreviousDay.blockCount === 0 && config.MEMPOOL.NETWORK === 'mainnet') { // We are done indexing
break;
}
@@ -359,9 +360,10 @@ class Mining {
// Add genesis block manually
if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
hashrates.push({
hashrateTimestamp: genesisTimestamp,
hashrateTimestamp: genesisTimestamp / 1000,
avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
poolId: null,
poolId: 0,
share: 1,
type: 'daily',
});
}
@@ -396,6 +398,15 @@ class Mining {
let currentDifficulty = 0;
let totalIndexed = 0;
if (indexedHeights[0] === false) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: 1231006505,
height: 0,
difficulty: 1.0,
adjustment: 0.0,
});
}
for (const block of blocks) {
if (block.difficulty !== currentDifficulty) {
if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed

1
backend/src/index.ts
View File

@@ -285,6 +285,7 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', routes.$getDifficultyAdjustments)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)

2
backend/src/repositories/BlocksRepository.ts
View File

@@ -436,7 +436,7 @@ class BlocksRepository {
}
if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
await this.$deleteBlocksFrom(blocks[idx - 1].height);
await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);

9
backend/src/repositories/DifficultyAdjustmentsRepository.ts
View File

@@ -1,4 +1,5 @@
import { Common } from '../api/common';
import config from '../config';
import DB from '../database';
import logger from '../logger';
import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
@@ -31,13 +32,19 @@ class DifficultyAdjustmentsRepository {
public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
interval = Common.getSqlInterval(interval);
let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment
let query = `SELECT
CAST(AVG(UNIX_TIMESTAMP(time)) as INT) as time,
CAST(AVG(height) AS INT) as height,
CAST(AVG(difficulty) as DOUBLE) as difficulty,
CAST(AVG(adjustment) as DOUBLE) as adjustment
FROM difficulty_adjustments`;
if (interval) {
query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${86400}`;
if (descOrder === true) {
query += ` ORDER BY time DESC`;
} else {

9
backend/src/repositories/HashratesRepository.ts
View File

@@ -1,5 +1,6 @@
import { escape } from 'mysql2';
import { Common } from '../api/common';
import config from '../config';
import DB from '../database';
import logger from '../logger';
import PoolsRepository from './PoolsRepository';
@@ -32,7 +33,9 @@ class HashratesRepository {
public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
interval = Common.getSqlInterval(interval);
let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
let query = `SELECT
CAST(AVG(UNIX_TIMESTAMP(hashrate_timestamp)) as INT) as timestamp,
CAST(AVG(avg_hashrate) as DOUBLE) as avgHashrate
FROM hashrates`;
if (interval) {
@@ -42,6 +45,7 @@ class HashratesRepository {
query += ` WHERE hashrates.type = 'daily'`;
}
query += ` GROUP BY UNIX_TIMESTAMP(hashrate_timestamp) DIV ${86400}`;
query += ` ORDER by hashrate_timestamp`;
try {
@@ -75,6 +79,9 @@ class HashratesRepository {
interval = Common.getSqlInterval(interval);
const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
if (topPoolsId.length === 0) {
return [];
}
let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
FROM hashrates

6
backend/src/repositories/PricesRepository.ts
View File

@@ -5,7 +5,11 @@ import { Prices } from '../tasks/price-updater';
class PricesRepository {
public async $savePrices(time: number, prices: Prices): Promise<void> {
try {
await DB.query(`INSERT INTO prices(time, avg_prices) VALUE (FROM_UNIXTIME(?), ?)`, [time, JSON.stringify(prices)]);
await DB.query(`
INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`,
[time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
);
} catch (e: any) {
logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;

2
backend/src/tasks/price-feeds/kraken-api.ts
View File

@@ -87,7 +87,7 @@ class KrakenApi implements PriceFeed {
}
if (Object.keys(priceHistory).length > 0) {
logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
}
}
}

21
backend/src/tasks/price-updater.ts
View File

@@ -176,7 +176,7 @@ class PriceUpdater {
++insertedCount;
}
if (insertedCount > 0) {
logger.info(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
}
// Insert Kraken weekly prices
@@ -205,23 +205,23 @@ class PriceUpdater {
try {
historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies));
} catch (e) {
logger.info(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
}
}
// Group them by timestamp and currency, for example
// grouped[123456789]['USD'] = [1, 2, 3, 4];
let grouped: Object = {};
const grouped: Object = {};
for (const historicalEntry of historicalPrices) {
for (const time in historicalEntry) {
if (existingPriceTimes.includes(parseInt(time, 10))) {
continue;
}
if (grouped[time] == undefined) {
if (grouped[time] === undefined) {
grouped[time] = {
USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: []
}
};
}
for (const currency of this.currencies) {
@@ -238,13 +238,20 @@ class PriceUpdater {
for (const time in grouped) {
const prices: Prices = this.getEmptyPricesObj();
for (const currency in grouped[time]) {
prices[currency] = Math.round((grouped[time][currency].reduce((partialSum, a) => partialSum + a, 0)) / grouped[time][currency].length);
if (grouped[time][currency].length === 0) {
continue;
}
prices[currency] = Math.round((grouped[time][currency].reduce(
(partialSum, a) => partialSum + a, 0)
) / grouped[time][currency].length);
}
await PricesRepository.$savePrices(parseInt(time, 10), prices);
++totalInserted;
}
logger.info(`Inserted ${totalInserted} hourly historical prices into the db`);
if (totalInserted > 0) {
logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`);
}
}
}

8
backend/src/utils/axios-query.ts
View File

@@ -50,10 +50,14 @@ export async function query(path): Promise<object | undefined> {
}
return data.data;
} catch (e) {
logger.err(`Could not connect to ${path}. Reason: ` + (e instanceof Error ? e.message : e));
logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
retry++;
}
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
if (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
}
}
logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
return undefined;
}