Merge branch 'master' into nymkappa/bugfix/handle-error-pool-hashrate

2022-07-06 21:54:14 +02:00
parent a5635a5020 dcf48dac85
commit 91fa84d825
42 changed files with 1613 additions and 876 deletions
--- a/backend/package-lock.json
+++ b/backend/package-lock.json
@@ -17,9 +17,9 @@
        "express": "^4.18.0",
        "mysql2": "2.3.3",
        "node-worker-threads-pool": "^1.5.1",
-        "socks-proxy-agent": "^6.2.0",
-        "typescript": "~4.7.2",
-        "ws": "~8.7.0"
+        "socks-proxy-agent": "~7.0.0",
+        "typescript": "~4.7.4",
+        "ws": "~8.8.0"
      },
      "devDependencies": {
        "@types/compression": "^1.7.2",
@@ -2734,9 +2734,9 @@
      }
    },
    "node_modules/socks-proxy-agent": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.0.tgz",
-      "integrity": "sha512-wWqJhjb32Q6GsrUqzuFkukxb/zzide5quXYcMVpIjxalDBBYy2nqKCFQ/9+Ie4dvOYSQdOk3hUlZSdzZOd3zMQ==",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz",
+      "integrity": "sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==",
      "dependencies": {
        "agent-base": "^6.0.2",
        "debug": "^4.3.3",
@@ -2950,9 +2950,9 @@
      "integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g=="
    },
    "node_modules/typescript": {
-      "version": "4.7.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.2.tgz",
-      "integrity": "sha512-Mamb1iX2FDUpcTRzltPxgWMKy3fhg0TN378ylbktPGPK/99KbDtMQ4W1hwgsbPAsG3a0xKa1vmw4VKZQbkvz5A==",
+      "version": "4.7.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
+      "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",
      "bin": {
        "tsc": "bin/tsc",
        "tsserver": "bin/tsserver"
@@ -3052,9 +3052,9 @@
      "dev": true
    },
    "node_modules/ws": {
-      "version": "8.7.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.7.0.tgz",
-      "integrity": "sha512-c2gsP0PRwcLFzUiA8Mkr37/MI7ilIlHQxaEAtd0uNMbVMoy8puJyafRlm0bV9MbGSabUPeLrRRaqIBcFcA2Pqg==",
+      "version": "8.8.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz",
+      "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==",
      "engines": {
        "node": ">=10.0.0"
      },
@@ -5098,9 +5098,9 @@
      }
    },
    "socks-proxy-agent": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.0.tgz",
-      "integrity": "sha512-wWqJhjb32Q6GsrUqzuFkukxb/zzide5quXYcMVpIjxalDBBYy2nqKCFQ/9+Ie4dvOYSQdOk3hUlZSdzZOd3zMQ==",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz",
+      "integrity": "sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==",
      "requires": {
        "agent-base": "^6.0.2",
        "debug": "^4.3.3",
@@ -5256,9 +5256,9 @@
      "integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g=="
    },
    "typescript": {
-      "version": "4.7.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.2.tgz",
-      "integrity": "sha512-Mamb1iX2FDUpcTRzltPxgWMKy3fhg0TN378ylbktPGPK/99KbDtMQ4W1hwgsbPAsG3a0xKa1vmw4VKZQbkvz5A=="
+      "version": "4.7.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
+      "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ=="
    },
    "unpipe": {
      "version": "1.0.0",
@@ -5333,9 +5333,9 @@
      "dev": true
    },
    "ws": {
-      "version": "8.7.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.7.0.tgz",
-      "integrity": "sha512-c2gsP0PRwcLFzUiA8Mkr37/MI7ilIlHQxaEAtd0uNMbVMoy8puJyafRlm0bV9MbGSabUPeLrRRaqIBcFcA2Pqg==",
+      "version": "8.8.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz",
+      "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==",
      "requires": {}
    },
    "yallist": {
--- a/backend/package.json
+++ b/backend/package.json
@@ -37,9 +37,9 @@
    "express": "^4.18.0",
    "mysql2": "2.3.3",
    "node-worker-threads-pool": "^1.5.1",
-    "socks-proxy-agent": "^6.2.0",
-    "typescript": "~4.7.2",
-    "ws": "~8.7.0"
+    "socks-proxy-agent": "~7.0.0",
+    "typescript": "~4.7.4",
+    "ws": "~8.8.0"
  },
  "devDependencies": {
    "@types/compression": "^1.7.2",
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -295,7 +295,8 @@ class Blocks {
      }
      logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
    } catch (e) {
-      logger.err(`Blocks summaries indexing failed. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
    }
  }

@@ -367,18 +368,12 @@ class Blocks {
      logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
      loadingIndicators.setProgress('block-indexing', 100);
    } catch (e) {
-      logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
      loadingIndicators.setProgress('block-indexing', 100);
-      return false;
+      throw e;
    }

-    const chainValid = await BlocksRepository.$validateChain();
-    if (!chainValid) {
-      indexer.reindex();
-      return false;
-    }
-
-    return true;
+    return await BlocksRepository.$validateChain();
  }

  public async $updateBlocks() {
--- a/backend/src/api/mining.ts
+++ b/backend/src/api/mining.ts
@@ -7,7 +7,9 @@ import logger from '../logger';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import { escape } from 'mysql2';
+import indexer from '../indexer';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import config from '../config';

 class Mining {
  constructor() {
@@ -263,6 +265,7 @@ class Mining {
      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
    } catch (e) {
      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
      throw e;
    }
  }
@@ -302,7 +305,7 @@ class Mining {
      while (toTimestamp > genesisTimestamp) {
        const fromTimestamp = toTimestamp - 86400000;

-        // Skip already indexed weeks
+        // Skip already indexed days
        if (indexedTimestamp.includes(toTimestamp / 1000)) {
          toTimestamp -= 86400000;
          ++totalIndexed;
@@ -313,7 +316,7 @@ class Mining {
        // we are currently indexing has complete data)
        const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
          null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
-        if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
+        if (blockStatsPreviousDay.blockCount === 0 && config.MEMPOOL.NETWORK === 'mainnet') { // We are done indexing
          break;
        }

@@ -357,9 +360,10 @@ class Mining {
      // Add genesis block manually
      if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
        hashrates.push({
-          hashrateTimestamp: genesisTimestamp,
+          hashrateTimestamp: genesisTimestamp / 1000,
          avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
-          poolId: null,
+          poolId: 0,
+          share: 1,
          type: 'daily',
        });
      }
@@ -374,6 +378,7 @@ class Mining {
      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
    } catch (e) {
      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
      throw e;
    }
  }
@@ -393,6 +398,15 @@ class Mining {
    let currentDifficulty = 0;
    let totalIndexed = 0;

+    if (indexedHeights[0] === false) {
+      await DifficultyAdjustmentsRepository.$saveAdjustments({
+        time: 1231006505,
+        height: 0,
+        difficulty: 1.0,
+        adjustment: 0.0,
+      });
+    }
+
    for (const block of blocks) {
      if (block.difficulty !== currentDifficulty) {
        if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -285,6 +285,7 @@ class Server {
        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', routes.$getDifficultyAdjustments)
        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
--- a/backend/src/indexer.ts
+++ b/backend/src/indexer.ts
@@ -39,6 +39,8 @@ class Indexer {
      const chainValid = await blocks.$generateBlockDatabase();
      if (chainValid === false) {
        // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
+        setTimeout(() => this.reindex(), 10000);
        this.indexerRunning = false;
        return;
      }
@@ -49,8 +51,9 @@ class Indexer {
      await mining.$generatePoolHashrateHistory();
      await blocks.$generateBlocksSummariesDatabase();
    } catch (e) {
-      this.reindex();
-      logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
+      this.indexerRunning = false;
+      logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
+      setTimeout(() => this.reindex(), 10000);
    }

    this.indexerRunning = false;
@@ -62,6 +65,7 @@ class Indexer {
      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
    } catch (e) {
      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
    }
  }
 }
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -436,7 +436,7 @@ class BlocksRepository {
        }

        if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
-          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
+          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
          await this.$deleteBlocksFrom(blocks[idx - 1].height);
          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
          await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
--- a/backend/src/repositories/DifficultyAdjustmentsRepository.ts
+++ b/backend/src/repositories/DifficultyAdjustmentsRepository.ts
@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
@@ -31,13 +32,19 @@ class DifficultyAdjustmentsRepository {
  public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
    interval = Common.getSqlInterval(interval);

-    let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment
+    let query = `SELECT 
+      CAST(AVG(UNIX_TIMESTAMP(time)) as INT) as time,
+      CAST(AVG(height) AS INT) as height,
+      CAST(AVG(difficulty) as DOUBLE) as difficulty,
+      CAST(AVG(adjustment) as DOUBLE) as adjustment
      FROM difficulty_adjustments`;

    if (interval) {
      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
    }

+    query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${86400}`;
+
    if (descOrder === true) {
      query += ` ORDER BY time DESC`;
    } else {
--- a/backend/src/repositories/HashratesRepository.ts
+++ b/backend/src/repositories/HashratesRepository.ts
@@ -1,5 +1,6 @@
 import { escape } from 'mysql2';
 import { Common } from '../api/common';
+import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import PoolsRepository from './PoolsRepository';
@@ -32,7 +33,9 @@ class HashratesRepository {
  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
    interval = Common.getSqlInterval(interval);

-    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
+    let query = `SELECT
+      CAST(AVG(UNIX_TIMESTAMP(hashrate_timestamp)) as INT) as timestamp,
+      CAST(AVG(avg_hashrate) as DOUBLE) as avgHashrate
      FROM hashrates`;

    if (interval) {
@@ -42,6 +45,7 @@ class HashratesRepository {
      query += ` WHERE hashrates.type = 'daily'`;
    }

+    query += ` GROUP BY UNIX_TIMESTAMP(hashrate_timestamp) DIV ${86400}`;
    query += ` ORDER by hashrate_timestamp`;

    try {