Bump version to v2.4.2-dev

Fix block predition graph x axis labels

.github/ISSUE_TEMPLATE.md

@@ -1,37 +0,0 @@
-<!--
-   SUPPORT REQUESTS: This is for reporting bugs in Mempool.
-   If you have a support request, please join our Keybase group:
-   https://keybase.io/team/mempool
--->
-
-### Description
-
-<!-- brief description of the bug -->
-
-#### Version
-
-<!-- commit id or version number -->
-
-### Steps to reproduce
-
-<!--if you can reliably reproduce the bug, list the steps here  -->
-
-### Expected behaviour
-
-<!--description of the expected behavior -->
-
-### Actual behaviour
-
-<!-- explain what happened instead of the expected behaviour -->
-
-### Screenshots
-
-<!--Screenshots if gui related, drag and drop to add to the issue -->
-
-#### Device or machine
-
-<!-- device/machine used, operating system -->
-
-#### Additional info
-
-<!-- Additional information useful for debugging (e.g. logs) -->

.github/ISSUE_TEMPLATE/00-bug-issue.md

@@ -0,0 +1,43 @@
+---
+name: 🐛 Bug Report
+about: Report bugs (no support requests, please)
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for reporting bugs in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the bug -->
+
+#### Version
+
+<!-- commit id or version number -->
+
+### Steps to reproduce
+
+<!-- if you can reliably reproduce the bug, list the steps here -->
+
+### Expected behaviour
+
+<!-- description of the expected behavior -->
+
+### Actual behaviour
+
+<!-- explain what happened instead of the expected behaviour -->
+
+### Screenshots
+
+<!-- Screenshots if gui related, drag and drop to add to the issue -->
+
+#### Device or machine
+
+<!-- device/machine used, operating system -->
+
+#### Additional info
+
+<!-- Additional information useful for debugging (e.g. logs) -->

.github/ISSUE_TEMPLATE/30-feature-request.md

@@ -0,0 +1,27 @@
+---
+name: ✨ Feature Request
+about: Request a feature or suggest other enhancements
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for requesting features in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the feature request -->
+
+### Problem to be solved
+
+<!-- description of the the problem you're having -->
+
+### Proposed solution
+
+<!-- explain how you think we should solve the problem -->
+
+#### Additional info
+
+<!-- Additional information useful for implementing (e.g. docs, links, etc.) -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🙋 Need help? Chat with us on Matrix
+    url: https://matrix.to/#/#mempool.support:bitcoin.kyoto
+    about: For support requests or general questions
+  - name: 🌐 Want to help with translations? Use Transifex
+    url: https://www.transifex.com/mempool/mempool
+    about: All translations work is done on Transifex

.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+- package-ecosystem: npm
+  directory: "/backend"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+- package-ecosystem: npm
+  directory: "/frontend"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+- package-ecosystem: docker
+  directory: "/docker/backend"
+  schedule:
+    interval: weekly
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+<!--
+Please do not open pull requests for translations.
+
+All translations work is done on Transifex:
+https://www.transifex.com/mempool/mempool
+-->

.github/workflows/ci.yml

@@ -0,0 +1,94 @@
+name: CI Pipeline for the Backend and Frontend
+
+on:
+  pull_request:
+    types: [opened, review_requested, synchronize]
+
+jobs:
+  backend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.16.0", "18.5.0"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Install (Prod dependencies only)
+        if: ${{ matrix.flavor == 'prod'}}
+        run: npm ci --omit=dev --omit=optional
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      #  - name: Test
+      #    run: npm run test
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+  frontend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.15.0", "18.5.0"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install (Prod dependencies only)
+        run: npm ci --omit=dev --omit=optional
+        if: ${{ matrix.flavor == 'prod'}}
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      # - name: Test
+      #   run: npm run test
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend

.github/workflows/cypress.yml

@@ -1,9 +1,11 @@
 name: Cypress Tests
 
-on: [push, pull_request]
-
+on:
+  pull_request:
+    types: [ opened, review_requested, synchronize ]
 jobs:
   cypress:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -18,63 +20,75 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2
         with:
-          node-version: 16.10.0
+          node-version: 16.15.0
           cache: 'npm'
           cache-dependency-path: frontend/package-lock.json
       - name: ${{ matrix.browser }} browser tests (Mempool)
-        uses: cypress-io/github-action@v2
+        uses: cypress-io/github-action@v4
         with:
+          tag: ${{ github.event_name }}
           working-directory: frontend
           build: npm run config:defaults:mempool
-          start: npm run start:local-prod
+          start: npm run start:local-staging
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
+          spec: |
+            cypress/e2e/mainnet/*.spec.ts
+            cypress/e2e/signet/*.spec.ts
+            cypress/e2e/testnet/*.spec.ts
           group: Tests on ${{ matrix.browser }} (Mempool)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
 
       - name: ${{ matrix.browser }} browser tests (Liquid)
-        uses: cypress-io/github-action@v2
+        uses: cypress-io/github-action@v4
         if: always()
         with:
+          tag: ${{ github.event_name }}
           working-directory: frontend
           build: npm run config:defaults:liquid
-          start: npm run start:local-prod
+          start: npm run start:local-staging
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
-          spec: cypress/integration/liquid/liquid.spec.ts
+          spec: |
+            cypress/e2e/liquid/liquid.spec.ts
+            cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
           group: Tests on ${{ matrix.browser }} (Liquid)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
 
       - name: ${{ matrix.browser }} browser tests (Bisq)
-        uses: cypress-io/github-action@v2
+        uses: cypress-io/github-action@v4
         if: always()
         with:
+          tag: ${{ github.event_name }}
           working-directory: frontend
           build: npm run config:defaults:bisq
-          start: npm run start:local-prod
+          start: npm run start:local-staging
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
-          spec: cypress/integration/bisq/bisq.spec.ts
+          spec: cypress/e2e/bisq/bisq.spec.ts
           group: Tests on ${{ matrix.browser }} (Bisq)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}

.github/workflows/on-tag.yml

@@ -11,6 +11,9 @@ on:
       - v[0-9]+.[0-9]+.[0-9]+
       - v[0-9]+.[0-9]+.[0-9]+-*
 
+permissions:
+  contents: read
+
 jobs:
   build:
     strategy:
@@ -35,24 +38,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@661fd3eb7f2f20d8c7c84bc2b0509efd7a826628 # v2
         id: cache
         with:
           path: /tmp/.buildx-cache

.gitignore

@@ -2,3 +2,4 @@ sitemap
 data
 docker-compose.yml
 backend/mempool-config.json
+*.swp

.nvmrc

@@ -1 +1 @@
-v16.10.0
+v16.16.0

CONTRIBUTING.md

@@ -0,0 +1,49 @@
+# Contributing to The Mempool Open Source Project
+
+Thank you for contributing to The Mempool Open Source Project managed by Mempool Space K.K. (“Mempool”).
+
+In order to clarify the intellectual property license granted with Contributions from any person or entity, Mempool must have a statement on file from each Contributor indicating their agreement to the Contributor License Agreement (“Agreement”). This license is for your protection as a Contributor as well as the protection of Mempool and its other contributors and users; it does not change your rights to use your own Contributions for any other purpose.
+
+When submitting a pull request for the first time, please create a file with a name like `/contributors/{github_username}.txt`, and in the content of that file indicate your agreement to the Contributor License Agreement terms below. An example of what that file should contain can be seen in wiz's agreement file. (This method of CLA "signing" is borrowed from Medium's open source project.)
+
+# Contributor License Agreement
+
+Last Updated: January 25, 2022
+
+By accepting this Agreement, You agree to the following terms and conditions for Your present and future Contributions submitted to Mempool. Except for the license granted herein to Mempool and recipients of software distributed by Mempool, You reserve all right, title, and interest in and to Your Contributions.
+
+### 1. Definitions
+
+“You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Mempool. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+“Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to Mempool for inclusion in, or documentation of, any of the products owned or managed by Mempool (“Work”). For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to Mempool or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Mempool for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as “Not a Contribution.”
+
+### 2. Grant of Copyright License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works.
+
+### 3. Grant of Patent License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
+
+### 4. Authority
+
+You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Mempool, or that your employer has executed a separate Corporate Contributor License Agreement with Mempool.
+
+### 5. Originality
+
+You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware, and which are associated with any part of Your Contributions.
+
+### 6. Support
+
+You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
+
+### 7. Third Party Contributions
+
+Should You wish to submit work that is not Your original creation, You may submit it to Mempool separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as “Submitted on behalf of a third-party: [named here]”.
+
+### 8. Notifications
+
+You agree to notify Mempool of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.
+
+EOF

GNUmakefile

@@ -25,8 +25,7 @@ help:
 .PHONY: init
 init:
 	@echo ''
-	mkdir -p   $(DATA) $(DATA)/mysql $(DATA)/mysql/db-scripts $(DATA)/mysql/data 
-	install -v mariadb-structure.sql $(DATA)/mysql/db-scripts
+	mkdir -p   $(DATA) $(DATA)/mysql $(DATA)/mysql/data 
 	#REF: https://github.com/mempool/mempool/blob/master/docker/README.md
 	cat docker/docker-compose.yml > docker-compose.yml
 	cat backend/mempool-config.sample.json > backend/mempool-config.json

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project
-Copyright (c) 2019-2021 The Mempool Open Source Project Developers
+Copyright (c) 2019-2022 The Mempool Open Source Project Developers
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of (at your option) either:

README.md

@@ -1,188 +1,33 @@
-# The Mempool Open Source Project™
+# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
-Mempool is the fully featured visualizer, explorer, and API service running on [mempool.space](https://mempool.space/), an open source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market to help our transition into a multi-layer ecosystem.
+Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
-![mempool](https://mempool.space/resources/screenshots/v2.3.0-dashboard.png)
+It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
 
-## Installation Methods
+![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
 
-Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi distro, all the way to an advanced high availability cluster of powerful servers for a production instance. We support the following installation methods, ranked in order from simple to advanced:
+# Installation Methods
 
-1) One-click installation on: [Umbrel](https://github.com/getumbrel/umbrel), [RaspiBlitz](https://github.com/rootzoll/raspiblitz), [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo), or [MyNode](https://github.com/mynodebtc/mynode).
-2) [Docker installation on Linux using docker-compose](https://github.com/mempool/mempool/tree/master/docker)
-3) [Manual installation on Linux or FreeBSD](https://github.com/mempool/mempool#manual-installation)
-4) [Production installation on a powerful FreeBSD server](https://github.com/mempool/mempool/tree/master/production)
-5) [High Availability cluster using powerful FreeBSD servers](https://github.com/mempool/mempool/tree/master/production#high-availability)
+Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 
 
-# Manual Installation
+**Most people should use a one-click install method.** Other install methods are meant for developers and others with experience managing servers. 
 
-The following instructions are for a manual installation on Linux or FreeBSD. The file and directory paths may need to be changed to match your OS.
+<a id="one-click-installation"></a>
+## One-Click Installation
 
-## Dependencies
+Mempool can be conveniently installed on the following full-node distros: 
+- [Umbrel](https://github.com/getumbrel/umbrel)
+- [RaspiBlitz](https://github.com/rootzoll/raspiblitz)
+- [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo)
+- [myNode](https://github.com/mynodebtc/mynode)
+- [Start9](https://github.com/Start9Labs/embassy-os)
 
-* [Bitcoin](https://github.com/bitcoin/bitcoin)
-* [Electrum](https://github.com/romanz/electrs)
-* [NodeJS](https://github.com/nodejs/node)
-* [MariaDB](https://github.com/mariadb/server)
-* [Nginx](https://github.com/nginx/nginx)
+**We highly recommend you deploy your own Mempool instance this way.** No matter which option you pick, you'll be able to get your own fully-sovereign instance of Mempool up quickly without needing to fiddle with any settings.
 
-## Mempool
+## Advanced Installation Methods
 
-Clone the mempool repo, and checkout the latest release tag:
-```bash
-  git clone https://github.com/mempool/mempool
-  cd mempool
-  latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
-  git checkout $latestrelease
-```
+Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
-## Bitcoin Core (bitcoind)
-
-Enable RPC and txindex in `bitcoin.conf`:
-```bash
-  rpcuser=mempool
-  rpcpassword=mempool
-  txindex=1
-```
-
-## MySQL
-
-Install MariaDB from OS package manager:
-```bash
-  # Linux
-  apt-get install mariadb-server mariadb-client
-
-  # macOS
-  brew install mariadb
-  mysql.server start
-```
-
-Create database and grant privileges:
-```bash
-  MariaDB [(none)]> drop database mempool;
-  Query OK, 0 rows affected (0.00 sec)
-
-  MariaDB [(none)]> create database mempool;
-  Query OK, 1 row affected (0.00 sec)
-
-  MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
-  Query OK, 0 rows affected (0.00 sec)
-```
-
-From the mempool repo's top-level folder, import the database structure:
-```bash
-  mysql -u mempool -p mempool < mariadb-structure.sql
-```
-
-## Mempool Backend
-Install mempool dependencies from npm and build the backend:
-
-```bash
-  # backend
-  cd backend
-  npm install --prod
-  npm run build
-```
-
-In the `backend` folder, make a copy of the sample config and modify it to fit your settings.
-
-```bash
-  cp mempool-config.sample.json mempool-config.json
-```
-
-Edit `mempool-config.json` to add your Bitcoin Core node RPC credentials:
-```bash
-{
-  "MEMPOOL": {
-    "NETWORK": "mainnet",
-    "BACKEND": "electrum",
-    "HTTP_PORT": 8999
-  },
-  "CORE_RPC": {
-    "HOST": "127.0.0.1",
-    "PORT": 8332,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
-  },
-  "ELECTRUM": {
-    "HOST": "127.0.0.1",
-    "PORT": 50002,
-    "TLS_ENABLED": true
-  },
-  "DATABASE": {
-    "ENABLED": true,
-    "HOST": "127.0.0.1",
-    "PORT": 3306,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "DATABASE": "mempool"
-  }
-}
-```
-
-Start the backend:
-
-```bash
-  npm run start
-```
-
-When it's running you should see output like this:
-
-```bash
-  Mempool updated in 0.189 seconds
-  Updating mempool
-  Mempool updated in 0.096 seconds
-  Updating mempool
-  Mempool updated in 0.099 seconds
-  Updating mempool
-  Calculated fee for transaction 1 / 10
-  Calculated fee for transaction 2 / 10
-  Calculated fee for transaction 3 / 10
-  Calculated fee for transaction 4 / 10
-  Calculated fee for transaction 5 / 10
-  Calculated fee for transaction 6 / 10
-  Calculated fee for transaction 7 / 10
-  Calculated fee for transaction 8 / 10
-  Calculated fee for transaction 9 / 10
-  Calculated fee for transaction 10 / 10
-  Mempool updated in 0.243 seconds
-  Updating mempool
-```
-
-## Mempool Frontend
-
-Install mempool dependencies from npm and build the frontend static HTML/CSS/JS:
-
-```bash
-  # frontend
-  cd frontend
-  npm install --prod
-  npm run build
-```
-
-Install the output into nginx webroot folder:
-
-```bash
-  sudo rsync -av --delete dist/mempool/ /var/www/
-```
-
-## nginx + certbot
-
-Install the supplied nginx.conf and nginx-mempool.conf in /etc/nginx
-
-```bash
-  # install nginx and certbot
-  apt-get install -y nginx python3-certbot-nginx
-
-  # install the mempool configuration for nginx
-  cp nginx.conf nginx-mempool.conf /etc/nginx/
-
-  # replace example.com with your domain name
-  certbot --nginx -d example.com
-
-```
-
-If everything went okay you should see the beautiful mempool :grin:
-
-If you get stuck on "loading blocks", this means the websocket can't connect.
-Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.
+- See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
+- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers.
+- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.

backend/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
+

backend/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

backend/.eslintrc

@@ -0,0 +1,33 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1
+  }
+}

backend/.gitignore

@@ -1,7 +1,10 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
-# production config
-mempool-config.json
+# production config and external assets
+*.json
+!mempool-config.sample.json
+
+icons.json
 
 # compiled output
 /dist

backend/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

backend/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "endOfLine": "lf",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

backend/README.md

@@ -0,0 +1,215 @@
+# Mempool Backend
+
+These instructions are mostly intended for developers. 
+
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
+
+See other ways to set up Mempool on [the main README](/../../#installation-methods).
+
+Jump to a section in this doc:
+- [Set Up the Backend](#setup)
+- [Development Tips](#development-tips)
+
+## Setup
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
+
+```
+git clone https://github.com/mempool/mempool
+cd mempool
+```
+
+Check out the latest release:
+
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Configure Bitcoin Core
+
+Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
+
+```
+txindex=1
+server=1
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+### 3. Configure Electrum Server
+
+[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
+
+**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
+
+### 4. Configure MariaDB
+
+_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
+
+Get MariaDB from your operating system's package manager:
+
+```
+# Debian, Ubuntu, etc.
+apt-get install mariadb-server mariadb-client
+
+# macOS
+brew install mariadb
+mysql.server start
+```
+
+Create a database and grant privileges:
+
+```
+MariaDB [(none)]> drop database mempool;
+Query OK, 0 rows affected (0.00 sec)
+
+MariaDB [(none)]> create database mempool;
+Query OK, 1 row affected (0.00 sec)
+
+MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+Query OK, 0 rows affected (0.00 sec)
+```
+
+### 5. Prepare Mempool Backend
+
+#### Build
+
+_Make sure to use Node.js 16.10 and npm 7._
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd backend
+npm install
+npm run build
+```
+
+#### Configure
+
+In the backend folder, make a copy of the sample config file:
+
+```
+cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` as needed. 
+
+In particular, make sure:
+- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
+- the correct `BACKEND` is specified in `MEMPOOL`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
+  - "none" if you're not using any Electrum Server
+
+### 6. Run Mempool Backend
+
+Run the Mempool backend:
+
+```
+npm run start
+```
+
+When it's running, you should see output like this:
+
+```
+Mempool updated in 0.189 seconds
+Updating mempool
+Mempool updated in 0.096 seconds
+Updating mempool
+Mempool updated in 0.099 seconds
+Updating mempool
+Calculated fee for transaction 1 / 10
+Calculated fee for transaction 2 / 10
+Calculated fee for transaction 3 / 10
+Calculated fee for transaction 4 / 10
+Calculated fee for transaction 5 / 10
+Calculated fee for transaction 6 / 10
+Calculated fee for transaction 7 / 10
+Calculated fee for transaction 8 / 10
+Calculated fee for transaction 9 / 10
+Calculated fee for transaction 10 / 10
+Mempool updated in 0.243 seconds
+Updating mempool
+```
+
+### 7. Set Up Mempool Frontend
+With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
+
+## Development Tips
+
+### Set Up Backend Watchers
+
+The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
+
+As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
+
+First, install `nodemon` and `ts-node`:
+
+```
+npm install -g ts-node nodemon
+```
+
+Then, run the watcher:
+
+```
+nodemon src/index.ts --ignore cache/ --ignore pools.json
+```
+
+`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
+
+### Useful Regtest Commands
+
+Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe223a
+
+Run bitcoind on regtest:
+   ```
+   bitcoind -regtest -rpcport=8332
+   ```
+
+Create a new wallet, if needed:
+   ```
+   bitcoin-cli -regtest -rpcport=8332 createwallet test
+   ```
+
+Load wallet (this command may take a while if you have lot of UTXOs):
+   ```
+   bitcoin-cli -regtest -rpcport=8332 loadwallet test
+   ```
+
+Get a new address:
+   ```
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   ```
+
+Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
+   ```
+   bitcoin-cli -regtest -rpcport=8332 generatetoaddress 101 $address
+   ```
+
+Send 0.1 BTC at 5 sat/vB to another address:
+   ```
+   ./src/bitcoin-cli -named -regtest -rpcport=8332 sendtoaddress address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress) amount=0.1 fee_rate=5
+   ```
+
+See more example of `sendtoaddress`:
+   ```
+   ./src/bitcoin-cli sendtoaddress # will print the help
+   ```
+
+Mini script to generate transactions with random TX fee-rate (between 1 to 100 sat/vB). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
+   ```
+   #!/bin/bash
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   for i in {1..1000000}
+   do
+     ./src/bitcoin-cli -regtest -rpcport=8332 -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1  1 100)
+   done
+   ```
+
+Generate block at regular interval (every 10 seconds in this example):
+   ```
+   watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
+   ```

backend/mempool-config.sample.json

@@ -12,8 +12,16 @@
     "BLOCK_WEIGHT_UNITS": 4000000,
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
-    "PRICE_FEED_UPDATE_INTERVAL": 3600,
-    "USE_SECOND_NODE_FOR_MINFEE": false
+    "INDEXING_BLOCKS_AMOUNT": 11000,
+    "BLOCKS_SUMMARIES_INDEXING": false,
+    "PRICE_FEED_UPDATE_INTERVAL": 600,
+    "USE_SECOND_NODE_FOR_MINFEE": false,
+    "EXTERNAL_ASSETS": [],
+    "EXTERNAL_MAX_RETRY": 1,
+    "EXTERNAL_RETRY_INTERVAL": 0,
+    "USER_AGENT": "mempool",
+    "STDOUT_LOG_MIN_PRIORITY": "debug",
+    "AUTOMATIC_BLOCK_REINDEXING": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -39,6 +47,7 @@
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "DATABASE": "mempool",
     "USERNAME": "mempool",
     "PASSWORD": "mempool"
@@ -57,5 +66,25 @@
   "BISQ": {
     "ENABLED": false,
     "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": false,
+    "USE_ONION": true,
+    "HOST": "127.0.0.1",
+    "PORT": 9050,
+    "USERNAME": "",
+    "PASSWORD": ""
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
+    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "https://mempool.space/api/v1",
+    "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
+    "LIQUID_API": "https://liquid.network/api/v1",
+    "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
+    "BISQ_URL": "https://bisq.markets/api",
+    "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
   }
 }

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.3.0-dev",
+  "version": "2.4.2-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -20,31 +20,37 @@
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
     "tsc": "./node_modules/typescript/bin/tsc",
     "build": "npm run tsc",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "lint": "./node_modules/.bin/eslint . --ext .ts",
+    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
+    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@mempool/bitcoin": "^3.0.3",
     "@mempool/electrum-client": "^1.1.7",
-    "@types/ws": "8.2.2",
-    "axios": "0.24.0",
+    "@types/node": "^16.11.41",
+    "axios": "~0.27.2",
     "bitcoinjs-lib": "6.0.1",
     "crypto-js": "^4.0.0",
-    "express": "^4.17.1",
-    "locutus": "^2.0.12",
+    "express": "^4.18.0",
     "mysql2": "2.3.3",
-    "node-worker-threads-pool": "^1.4.3",
-    "typescript": "4.4.4",
-    "ws": "8.3.0"
+    "node-worker-threads-pool": "^1.5.1",
+    "socks-proxy-agent": "~7.0.0",
+    "typescript": "~4.7.4",
+    "ws": "~8.8.0"
   },
   "devDependencies": {
-    "@types/compression": "^1.0.1",
-    "@types/express": "^4.17.2",
-    "@types/locutus": "^0.0.6",
-    "tslint": "^6.1.0"
+    "@types/compression": "^1.7.2",
+    "@types/crypto-js": "^4.1.1",
+    "@types/express": "^4.17.13",
+    "@types/ws": "~8.5.3",
+    "@typescript-eslint/eslint-plugin": "^5.30.5",
+    "@typescript-eslint/parser": "^5.30.5",
+    "eslint": "^8.19.0",
+    "eslint-config-prettier": "^8.5.0",
+    "prettier": "^2.7.1"
   }
 }

backend/src/api/backend-info.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs';
 import * as os from 'os';
 import logger from '../logger';
 import { IBackendInfo } from '../mempool.interfaces';
+const { spawnSync } = require('child_process');
 
 class BackendInfo {
   private gitCommitHash = '';
@@ -27,10 +28,23 @@ class BackendInfo {
   }
 
   private setLatestCommitHash(): void {
-    try {
-      this.gitCommitHash = fs.readFileSync('../.git/refs/heads/master').toString().trim();
-    } catch (e) {
-      logger.err('Could not load git commit info: ' + (e instanceof Error ? e.message : e));
+    //TODO: share this logic with `generate-config.js`
+    if (process.env.DOCKER_COMMIT_HASH) {
+      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
+    } else {
+      try {
+        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+        if (!gitRevParse.error) {
+          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+          this.gitCommitHash = output ? output : '?';
+        } else if (gitRevParse.error.code === 'ENOENT') {
+          console.log('git not found, cannot parse git hash');
+          this.gitCommitHash = '?';
+        }
+      } catch (e: any) {
+        console.log('Could not load git commit info: ' + e.message);
+        this.gitCommitHash = '?';
+      }
     }
   }
 

backend/src/api/bisq/bisq.ts

@@ -1,10 +1,14 @@
 import config from '../../config';
 import * as fs from 'fs';
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
+import * as http from 'http';
+import * as https from 'https';
+import { SocksProxyAgent } from 'socks-proxy-agent';
 import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
 import { Common } from '../common';
 import { BlockExtended } from '../../mempool.interfaces';
 import { StaticPool } from 'node-worker-threads-pool';
+import backendInfo from '../backend-info';
 import logger from '../../logger';
 
 class Bisq {
@@ -35,7 +39,13 @@ class Bisq {
   constructor() {}
 
   startBisqService(): void {
-    this.checkForBisqDataFolder();
+    try {
+      this.checkForBisqDataFolder();
+    } catch (e) {
+      logger.info('Retrying to start bisq service in 3 minutes');
+      setTimeout(this.startBisqService.bind(this), 180000);
+      return;
+    }
     this.loadBisqDumpFile();
     setInterval(this.updatePrice.bind(this), 1000 * 60 * 60);
     this.updatePrice();
@@ -90,7 +100,7 @@ class Bisq {
   private checkForBisqDataFolder() {
     if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
       logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      return process.exit(1);
+      throw new Error(`Cannot load BISQ ${Bisq.BLOCKS_JSON_FILE_PATH} file`);
     }
   }
 
@@ -137,12 +147,59 @@ class Bisq {
       }, 2000);
     });
   }
+  private async updatePrice() {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
+    const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+    let retry = 0;
 
-  private updatePrice() {
-    axios.get<BisqTrade[]>('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 })
-      .then((response) => {
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
+          throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
+        }
         const prices: number[] = [];
-        response.data.forEach((trade) => {
+        data.data.forEach((trade) => {
           prices.push(parseFloat(trade.price) * 100000000);
         });
         prices.sort((a, b) => a - b);
@@ -150,19 +207,24 @@ class Bisq {
         if (this.priceUpdateCallbackFunction) {
           this.priceUpdateCallbackFunction(this.price);
         }
-    }).catch((err) => {
-      logger.err('Error updating Bisq market price: ' + err);
-    });
+        logger.debug('Successfully updated Bisq market price');
+        break;
+      } catch (e) {
+        logger.err('Error updating Bisq market price: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
+      }
+    }
   }
 
   private async loadBisqDumpFile(): Promise<void> {
+    this.allBlocks = [];
     try {
-      const data = await this.loadData();
-      await this.loadBisqBlocksDump(data);
+      await this.loadData();
       this.buildIndex();
       this.calculateStats();
     } catch (e) {
-      logger.info('loadBisqDumpFile() error.' + (e instanceof Error ? e.message : e));
+      logger.info('Cannot load bisq dump file because: ' + (e instanceof Error ? e.message : e));
     }
   }
 
@@ -241,36 +303,61 @@ class Bisq {
     };
   }
 
-  private async loadBisqBlocksDump(cacheData: string): Promise<void> {
-    const start = new Date().getTime();
-    if (cacheData && cacheData.length !== 0) {
-      logger.debug('Processing Bisq data dump...');
-      const data: BisqBlocks = await this.jsonParsePool.exec(cacheData);
-      if (data.blocks && data.blocks.length !== this.allBlocks.length) {
-        this.allBlocks = data.blocks;
-        this.allBlocks.reverse();
-        this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
-        this.latestBlockHeight = data.chainHeight;
-        const time = new Date().getTime() - start;
-        logger.debug('Bisq dump processed in ' + time + ' ms (worker thread)');
-      } else {
-        throw new Error(`Bisq dump didn't contain any blocks`);
-      }
+  private async loadData(): Promise<any> {
+    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
+      throw new Error(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
     }
-  }
 
-  private loadData(): Promise<string> {
-    return new Promise((resolve, reject) => {
-      if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
-        return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
-      }
-      fs.readFile(Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
-        if (err) {
-          reject(err);
-        }
-        resolve(data);
-      });
+    const readline = require('readline');
+    const events = require('events');
+
+    const rl = readline.createInterface({
+      input: fs.createReadStream(Bisq.BLOCKS_JSON_FILE_PATH),
+      crlfDelay: Infinity
     });
+
+    let blockBuffer = '';
+    let readingBlock = false;
+    let lineCount = 1;
+    const start = new Date().getTime();
+
+    logger.debug('Processing Bisq data dump...');
+
+    rl.on('line', (line) => {
+      if (lineCount === 2) {
+        line = line.replace('  "chainHeight": ', '');
+        this.latestBlockHeight = parseInt(line, 10);
+      }
+
+      if (line === '    {') {
+        readingBlock = true;
+      } else if (line === '    },') {
+        blockBuffer += '}';
+        try {
+          const block: BisqBlock =  JSON.parse(blockBuffer);
+          this.allBlocks.push(block);
+          readingBlock = false;
+          blockBuffer = '';
+        } catch (e) {
+          logger.debug(blockBuffer);
+          throw Error(`Unable to parse Bisq data dump at line ${lineCount}` + (e instanceof Error ? e.message : e));
+        }
+      }
+
+      if (readingBlock === true) {
+        blockBuffer += line;
+      }
+
+      ++lineCount;
+    });
+
+    await events.once(rl, 'close');
+
+    this.allBlocks.reverse();
+    this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
+
+    const time = new Date().getTime() - start;
+    logger.debug('Bisq dump processed in ' + time + ' ms');
   }
 }
 

backend/src/api/bisq/markets-api.ts

@@ -1,7 +1,7 @@
 import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
   Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
 
-import * as datetime from 'locutus/php/datetime';
+const strtotime = require('./strtotime');
 
 class BisqMarketsApi {
   private cryptoCurrencyData: Currency[] = [];
@@ -312,7 +312,7 @@ class BisqMarketsApi {
 
   getTickerFromMarket(market: string): Ticker | null {
     let ticker: Ticker;
-    const timestamp_from = datetime.strtotime('-24 hour');
+    const timestamp_from = strtotime('-24 hour');
     const timestamp_to = new Date().getTime() / 1000;
     const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
       undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
@@ -638,13 +638,13 @@ class BisqMarketsApi {
         case 'half_day':
             return (ts - (ts % (3600 * 12)));
         case 'day':
-            return datetime.strtotime('midnight today', ts);
+            return strtotime('midnight today', ts);
         case 'week':
-            return datetime.strtotime('midnight sunday last week', ts);
+            return strtotime('midnight sunday last week', ts);
         case 'month':
-            return datetime.strtotime('midnight first day of this month', ts);
+            return strtotime('midnight first day of this month', ts);
         case 'year':
-            return datetime.strtotime('midnight first day of january', ts);
+            return strtotime('midnight first day of january', ts);
         default:
             throw new Error('Unsupported interval: ' + interval);
     }

backend/src/api/bisq/markets.ts

@@ -26,7 +26,13 @@ class Bisq {
   constructor() {}
 
   startBisqService(): void {
-    this.checkForBisqDataFolder();
+    try {
+      this.checkForBisqDataFolder();
+    } catch (e) {
+      logger.info('Retrying to start bisq service (markets) in 3 minutes');
+      setTimeout(this.startBisqService.bind(this), 180000);
+      return;
+    }
     this.loadBisqDumpFile();
     this.startBisqDirectoryWatcher();
   }
@@ -34,7 +40,7 @@ class Bisq {
   private checkForBisqDataFolder() {
     if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
       logger.err(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      return process.exit(1);
+      throw new Error(`Cannot load BISQ ${Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency} file`);
     }
   }
 

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -2,8 +2,9 @@ import { IEsploraApi } from './esplora-api.interface';
 
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
-  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
+  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
   $getBlockHeightTip(): Promise<number>;
+  $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
@@ -12,6 +13,8 @@ export interface AbstractBitcoinApi {
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];
   $sendRawTransaction(rawTransaction: string): Promise<string>;
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
+  $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
 }
 export interface BitcoinRpcCredentials {
   host: string;

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -4,6 +4,7 @@ export namespace IBitcoinApi {
     size: number;                    //  (numeric) Current tx count
     bytes: number;                   //  (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
     usage: number;                   //  (numeric) Total memory usage for the mempool
+    total_fee: number;               //  (numeric) Total fees of transactions in the mempool
     maxmempool: number;              //  (numeric) Maximum memory usage for the mempool
     mempoolminfee: number;           //  (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
     minrelaytxfee: number;           //  (numeric) Current minimum relay fee for transactions
@@ -72,6 +73,14 @@ export namespace IBitcoinApi {
     time: number;                    //  (numeric) Same as blocktime
   }
 
+  export interface VerboseBlock extends Block {
+    tx: VerboseTransaction[];        // The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 "tx" result
+  }
+
+  export interface VerboseTransaction extends Transaction {
+    fee?: number;                   //  (numeric) The transaction fee in BTC, omitted if block undo data is not available
+  }
+
   export interface Vin {
     txid?: string;                   //  (string) The transaction id
     vout?: number;                   //  (string)

backend/src/api/bitcoin/bitcoin-api.ts

@@ -14,33 +14,61 @@ class BitcoinApi implements AbstractBitcoinApi {
     this.bitcoindClient = bitcoinClient;
   }
 
-  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
+  static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
+    return {
+      id: block.hash,
+      height: block.height,
+      version: block.version,
+      timestamp: block.time,
+      bits: parseInt(block.bits, 16),
+      nonce: block.nonce,
+      difficulty: block.difficulty,
+      merkle_root: block.merkleroot,
+      tx_count: block.nTx,
+      size: block.size,
+      weight: block.weight,
+      previousblockhash: block.previousblockhash,
+    };
+  }
+
+
+  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
     const txInMempool = mempool.getMempool()[txId];
     if (txInMempool && addPrevout) {
       return this.$addPrevouts(txInMempool);
     }
 
-    // Special case to fetch the Coinbase transaction
-    if (txId === '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b') {
-      return this.$returnCoinbaseTransaction();
-    }
-
     return this.bitcoindClient.getRawTransaction(txId, true)
       .then((transaction: IBitcoinApi.Transaction) => {
         if (skipConversion) {
           transaction.vout.forEach((vout) => {
-            vout.value = vout.value * 100000000;
+            vout.value = Math.round(vout.value * 100000000);
           });
           return transaction;
         }
-        return this.$convertTransaction(transaction, addPrevout);
+        return this.$convertTransaction(transaction, addPrevout, lazyPrevouts);
+      })
+      .catch((e: Error) => {
+        if (e.message.startsWith('The genesis block coinbase')) {
+          return this.$returnCoinbaseTransaction();
+        }
+        throw e;
       });
   }
 
   $getBlockHeightTip(): Promise<number> {
     return this.bitcoindClient.getChainTips()
-      .then((result: IBitcoinApi.ChainTips[]) => result[0].height);
+      .then((result: IBitcoinApi.ChainTips[]) => {
+        return result.find(tip => tip.status === 'active')!.height;
+      });
+  }
+
+  $getBlockHashTip(): Promise<string> {
+    return this.bitcoindClient.getChainTips()
+      .then((result: IBitcoinApi.ChainTips[]) => {
+        return result.find(tip => tip.status === 'active')!.hash;
+      });
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
@@ -67,7 +95,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     }
 
     return this.bitcoindClient.getBlock(hash)
-      .then((block: IBitcoinApi.Block) => this.convertBlock(block));
+      .then((block: IBitcoinApi.Block) => BitcoinApi.convertBlock(block));
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {
@@ -83,26 +111,58 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   $getAddressPrefix(prefix: string): string[] {
-    const found: string[] = [];
+    const found: { [address: string]: string } = {};
     const mp = mempool.getMempool();
     for (const tx in mp) {
       for (const vout of mp[tx].vout) {
         if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
-          found.push(vout.scriptpubkey_address);
-          if (found.length >= 10) {
-            return found;
+          found[vout.scriptpubkey_address] = '';
+          if (Object.keys(found).length >= 10) {
+            return Object.keys(found);
           }
         }
       }
     }
-    return found;
+    return Object.keys(found);
   }
 
   $sendRawTransaction(rawTransaction: string): Promise<string> {
     return this.bitcoindClient.sendRawTransaction(rawTransaction);
   }
 
-  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
+  async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    const outSpends: IEsploraApi.Outspend[] = [];
+    const tx = await this.$getRawTransaction(txId, true, false);
+    for (let i = 0; i < tx.vout.length; i++) {
+      if (tx.status && tx.status.block_height === 0) {
+        outSpends.push({
+          spent: false
+        });
+      } else {
+        const txOut = await this.bitcoindClient.getTxOut(txId, i);
+        outSpends.push({
+          spent: txOut === null,
+        });
+      }
+    }
+    return outSpends;
+  }
+
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
+  $getEstimatedHashrate(blockHeight: number): Promise<number> {
+    // 120 is the default block span in Core
+    return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
+  }
+
+  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
     let esploraTransaction: IEsploraApi.Transaction = {
       txid: transaction.txid,
       version: transaction.version,
@@ -117,11 +177,11 @@ class BitcoinApi implements AbstractBitcoinApi {
 
     esploraTransaction.vout = transaction.vout.map((vout) => {
       return {
-        value: vout.value * 100000000,
+        value: Math.round(vout.value * 100000000),
         scriptpubkey: vout.scriptPubKey.hex,
         scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
           : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
-        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.asm) : '',
+        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
         scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
       };
     });
@@ -131,7 +191,7 @@ class BitcoinApi implements AbstractBitcoinApi {
         is_coinbase: !!vin.coinbase,
         prevout: null,
         scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
-        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.asm) || '',
+        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
         sequence: vin.sequence,
         txid: vin.txid || '',
         vout: vin.vout || 0,
@@ -148,35 +208,15 @@ class BitcoinApi implements AbstractBitcoinApi {
       };
     }
 
-    if (transaction.confirmations) {
-      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
-    } else {
+    if (addPrevout) {
+      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
+    } else if (!transaction.confirmations) {
       esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
-      if (addPrevout) {
-        esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
-      }
     }
 
     return esploraTransaction;
   }
 
-  private convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
-    return {
-      id: block.hash,
-      height: block.height,
-      version: block.version,
-      timestamp: block.time,
-      bits: parseInt(block.bits, 16),
-      nonce: block.nonce,
-      difficulty: block.difficulty,
-      merkle_root: block.merkleroot,
-      tx_count: block.nTx,
-      size: block.size,
-      weight: block.weight,
-      previousblockhash: block.previousblockhash,
-    };
-  }
-
   private translateScriptPubKeyType(outputType: string): string {
     const map = {
       'pubkey': 'p2pk',
@@ -186,13 +226,14 @@ class BitcoinApi implements AbstractBitcoinApi {
       'witness_v0_scripthash': 'v0_p2wsh',
       'witness_v1_taproot': 'v1_p2tr',
       'nonstandard': 'nonstandard',
+      'multisig': 'multisig',
       'nulldata': 'op_return'
     };
 
     if (map[outputType]) {
       return map[outputType];
     } else {
-      return '';
+      return 'unknown';
     }
   }
 
@@ -209,7 +250,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     } else {
       mempoolEntry = await this.$getMempoolEntry(transaction.txid);
     }
-    transaction.fee = mempoolEntry.fees.base * 100000000;
+    transaction.fee = Math.round(mempoolEntry.fees.base * 100000000);
     return transaction;
   }
 
@@ -218,7 +259,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       if (vin.prevout) {
         continue;
       }
-      const innerTx = await this.$getRawTransaction(vin.txid, false);
+      const innerTx = await this.$getRawTransaction(vin.txid, false, false);
       vin.prevout = innerTx.vout[vin.vout];
       this.addInnerScriptsToVin(vin);
     }
@@ -226,12 +267,14 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   protected $returnCoinbaseTransaction(): Promise<IEsploraApi.Transaction> {
-    return this.bitcoindClient.getBlock('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', 2)
-      .then((block: IBitcoinApi.Block) => {
-        return this.$convertTransaction(Object.assign(block.tx[0], {
-          confirmations: blocks.getCurrentBlockHeight() + 1,
-          blocktime: 1231006505 }), false);
-      });
+    return this.bitcoindClient.getBlockHash(0).then((hash: string) =>
+      this.bitcoindClient.getBlock(hash, 2)
+        .then((block: IBitcoinApi.Block) => {
+          return this.$convertTransaction(Object.assign(block.tx[0], {
+            confirmations: blocks.getCurrentBlockHeight() + 1,
+            blocktime: block.time }), false);
+        })
+    );
   }
 
   private $getMempoolEntry(txid: string): Promise<IBitcoinApi.MempoolEntry> {
@@ -242,42 +285,95 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getRawMemPool(true);
   }
 
-  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
+
+  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
     if (transaction.vin[0].is_coinbase) {
       transaction.fee = 0;
       return transaction;
     }
     let totalIn = 0;
-    for (const vin of transaction.vin) {
-      const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout);
-      if (addPrevout) {
-        vin.prevout = innerTx.vout[vin.vout];
-        this.addInnerScriptsToVin(vin);
+
+    for (let i = 0; i < transaction.vin.length; i++) {
+      if (lazyPrevouts && i > 12) {
+        transaction.vin[i].lazy = true;
+        continue;
       }
-      totalIn += innerTx.vout[vin.vout].value;
+      const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
+      transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
+      this.addInnerScriptsToVin(transaction.vin[i]);
+      totalIn += innerTx.vout[transaction.vin[i].vout].value;
+    }
+    if (lazyPrevouts && transaction.vin.length > 12) {
+      transaction.fee = -1;
+    } else {
+      const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
+      transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
     }
-    const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
-    transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
     return transaction;
   }
 
-  private convertScriptSigAsm(str: string): string {
-    const a = str.split(' ');
+  private convertScriptSigAsm(hex: string): string {
+    const buf = Buffer.from(hex, 'hex');
+
     const b: string[] = [];
-    a.forEach((chunk) => {
-      if (chunk.substr(0, 3) === 'OP_') {
-        chunk = chunk.replace(/^OP_(\d+)/, 'OP_PUSHNUM_$1');
-        chunk = chunk.replace('OP_CHECKSEQUENCEVERIFY', 'OP_CSV');
-        b.push(chunk);
-      } else {
-        chunk = chunk.replace('[ALL]', '01');
-        if (chunk === '0') {
-          b.push('OP_0');
+
+    let i = 0;
+    while (i < buf.length) {
+      const op = buf[i];
+      if (op >= 0x01 && op <= 0x4e) {
+        i++;
+        let push: number;
+        if (op === 0x4c) {
+          push = buf.readUInt8(i);
+          b.push('OP_PUSHDATA1');
+          i += 1;
+        } else if (op === 0x4d) {
+          push = buf.readUInt16LE(i);
+          b.push('OP_PUSHDATA2');
+          i += 2;
+        } else if (op === 0x4e) {
+          push = buf.readUInt32LE(i);
+          b.push('OP_PUSHDATA4');
+          i += 4;
         } else {
-          b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk);
+          push = op;
+          b.push('OP_PUSHBYTES_' + push);
         }
+
+        const data = buf.slice(i, i + push);
+        if (data.length !== push) {
+          break;
+        }
+
+        b.push(data.toString('hex'));
+        i += data.length;
+      } else {
+        if (op === 0x00) {
+          b.push('OP_0');
+        } else if (op === 0x4f) {
+          b.push('OP_PUSHNUM_NEG1');
+        } else if (op === 0xb1) {
+          b.push('OP_CLTV');
+        } else if (op === 0xb2) {
+          b.push('OP_CSV');
+        } else if (op === 0xba) {
+          b.push('OP_CHECKSIGADD');
+        } else {
+          const opcode = bitcoinjs.script.toASM([ op ]);
+          if (opcode && op < 0xfd) {
+            if (/^OP_(\d+)$/.test(opcode)) {
+              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
+            } else {
+              b.push(opcode);
+            }
+          } else {
+            b.push('OP_RETURN_' + op);
+          }
+        }
+        i += 1;
       }
-    });
+    }
+
     return b.join(' ');
   }
 
@@ -288,21 +384,21 @@ class BitcoinApi implements AbstractBitcoinApi {
 
     if (vin.prevout.scriptpubkey_type === 'p2sh') {
       const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
-      vin.inner_redeemscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(redeemScript, 'hex')));
+      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
       if (vin.witness && vin.witness.length > 2) {
         const witnessScript = vin.witness[vin.witness.length - 1];
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
       }
     }
 
     if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
       const witnessScript = vin.witness[vin.witness.length - 1];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
     }
 
     if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) {
       const witnessScript = vin.witness[vin.witness.length - 2];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
     }
   }
 

backend/src/api/bitcoin/bitcoin-client.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-import * as bitcoin from '@mempool/bitcoin';
+const bitcoin = require('../../rpc-api/index');
 import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
 
 const nodeRpcCredentials: BitcoinRpcCredentials = {

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-import * as bitcoin from '@mempool/bitcoin';
+const bitcoin = require('../../rpc-api/index');
 import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
 
 const nodeRpcCredentials: BitcoinRpcCredentials = {

backend/src/api/bitcoin/electrum-api.ts

@@ -1,12 +1,11 @@
 import config from '../../config';
+import Client from '@mempool/electrum-client';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import { IElectrumApi } from './electrum-api.interface';
 import BitcoinApi from './bitcoin-api';
 import logger from '../../logger';
-import * as ElectrumClient from '@mempool/electrum-client';
-import * as sha256 from 'crypto-js/sha256';
-import * as hexEnc from 'crypto-js/enc-hex';
+import crypto from "crypto-js";
 import loadingIndicators from '../loading-indicators';
 import memoryCache from '../memory-cache';
 
@@ -26,7 +25,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       onLog: (str) => { logger.debug(str); },
     };
 
-    this.electrumClient = new ElectrumClient(
+    this.electrumClient = new Client(
       config.ELECTRUM.PORT,
       config.ELECTRUM.HOST,
       config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
@@ -35,7 +34,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     );
 
     this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
-      .then(() => {})
+      .then(() => { })
       .catch((err) => {
         logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
       });
@@ -95,7 +94,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-     return [];
+      return [];
     }
 
     try {
@@ -144,8 +143,8 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   }
 
   private encodeScriptHash(scriptPubKey: string): string {
-    const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey)));
-    return addrScripthash.match(/.{2}/g).reverse().join('');
+    const addrScripthash = crypto.enc.Hex.stringify(crypto.SHA256(crypto.enc.Hex.parse(scriptPubKey)));
+    return addrScripthash!.match(/.{2}/g)!.reverse().join('');
   }
 
 }

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -33,6 +33,8 @@ export namespace IEsploraApi {
     // Elements
     is_pegin?: boolean;
     issuance?: Issuance;
+    // Custom
+    lazy?: boolean;
   }
 
   interface Issuance {
@@ -113,9 +115,9 @@ export namespace IEsploraApi {
 
   export interface Outspend {
     spent: boolean;
-    txid: string;
-    vin: number;
-    status: Status;
+    txid?: string;
+    vin?: number;
+    status?: Status;
   }
 
   export interface Asset {

backend/src/api/bitcoin/esplora-api.ts

@@ -25,6 +25,11 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
+  $getBlockHashTip(): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
   $getTxIdsForBlock(hash: string): Promise<string[]> {
     return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
@@ -60,6 +65,20 @@ class ElectrsApi implements AbstractBitcoinApi {
   $sendRawTransaction(rawTransaction: string): Promise<string> {
     throw new Error('Method not implemented.');
   }
+
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
 }
 
 export default ElectrsApi;

backend/src/api/blocks.ts

@@ -2,14 +2,30 @@ import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, TransactionExtended } from '../mempool.interfaces';
+import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
 import bitcoinClient from './bitcoin/bitcoin-client';
+import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import poolsRepository from '../repositories/PoolsRepository';
+import blocksRepository from '../repositories/BlocksRepository';
+import loadingIndicators from './loading-indicators';
+import BitcoinApi from './bitcoin/bitcoin-api';
+import { prepareBlock } from '../utils/blocks-utils';
+import BlocksRepository from '../repositories/BlocksRepository';
+import HashratesRepository from '../repositories/HashratesRepository';
+import indexer from '../indexer';
+import poolsParser from './pools-parser';
+import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import mining from './mining';
+import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import difficultyAdjustment from './difficulty-adjustment';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
+  private blockSummaries: BlockSummary[] = [];
   private currentBlockHeight = 0;
   private currentDifficulty = 0;
   private lastDifficultyAdjustmentTime = 0;
@@ -26,15 +42,352 @@ class Blocks {
     this.blocks = blocks;
   }
 
+  public getBlockSummaries(): BlockSummary[] {
+    return this.blockSummaries;
+  }
+
+  public setBlockSummaries(blockSummaries: BlockSummary[]) {
+    this.blockSummaries = blockSummaries;
+  }
+
   public setNewBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void) {
     this.newBlockCallbacks.push(fn);
   }
 
+  /**
+   * Return the list of transaction for a block
+   * @param blockHash
+   * @param blockHeight
+   * @param onlyCoinbase - Set to true if you only need the coinbase transaction
+   * @returns Promise<TransactionExtended[]>
+   */
+  private async $getTransactionsExtended(
+    blockHash: string,
+    blockHeight: number,
+    onlyCoinbase: boolean,
+    quiet: boolean = false,
+  ): Promise<TransactionExtended[]> {
+    const transactions: TransactionExtended[] = [];
+    const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+
+    const mempool = memPool.getMempool();
+    let transactionsFound = 0;
+    let transactionsFetched = 0;
+
+    for (let i = 0; i < txIds.length; i++) {
+      if (mempool[txIds[i]]) {
+        // We update blocks before the mempool (index.ts), therefore we can
+        // optimize here by directly fetching txs in the "outdated" mempool
+        transactions.push(mempool[txIds[i]]);
+        transactionsFound++;
+      } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
+        // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
+        if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
+          logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
+        }
+        try {
+          const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
+          transactions.push(tx);
+          transactionsFetched++;
+        } catch (e) {
+          if (i === 0) {
+            const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
+            logger.err(msg);
+            throw new Error(msg);
+          } else {
+            logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+          }
+        }
+      }
+
+      if (onlyCoinbase === true) {
+        break; // Fetch the first transaction and exit
+      }
+    }
+
+    transactions.forEach((tx) => {
+      if (!tx.cpfpChecked) {
+        Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent
+      }
+    });
+
+    if (!quiet) {
+      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+    }
+
+    return transactions;
+  }
+
+  /**
+   * Return a block summary (list of stripped transactions)
+   * @param block
+   * @returns BlockSummary
+   */
+  private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
+    const stripped = block.tx.map((tx) => {
+      return {
+        txid: tx.txid,
+        vsize: tx.vsize,
+        fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
+        value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
+      };
+    });
+
+    return {
+      id: block.hash,
+      transactions: stripped
+    };
+  }
+
+  /**
+   * Return a block with additional data (reward, coinbase, fees...)
+   * @param block
+   * @param transactions
+   * @returns BlockExtended
+   */
+  private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
+    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
+    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
+
+    if (block.height === 0) {
+      blockExtended.extras.medianFee = 0; // 50th percentiles
+      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      blockExtended.extras.totalFees = 0;
+      blockExtended.extras.avgFee = 0;
+      blockExtended.extras.avgFeeRate = 0;
+    } else {
+      const stats = await bitcoinClient.getBlockStats(block.id, [
+        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
+      ]);
+      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      blockExtended.extras.totalFees = stats.totalfee;
+      blockExtended.extras.avgFee = stats.avgfee;
+      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+    }
+
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+      let pool: PoolTag;
+      if (blockExtended.extras?.coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      } else {
+        if (config.DATABASE.ENABLED === true) {
+          pool = await poolsRepository.$getUnknownPool();
+        } else {
+          pool = poolsParser.unknownPool;
+        }
+      }
+
+      if (!pool) { // We should never have this situation in practise
+        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+          `Check your "pools" table entries`);
+        return blockExtended;
+      }
+
+      blockExtended.extras.pool = {
+        id: pool.id,
+        name: pool.name,
+        slug: pool.slug,
+      };
+    }
+
+    return blockExtended;
+  }
+
+  /**
+   * Try to find which miner found the block
+   * @param txMinerInfo
+   * @returns
+   */
+  private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
+    if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
+      if (config.DATABASE.ENABLED === true) {
+        return await poolsRepository.$getUnknownPool();
+      } else {
+        return poolsParser.unknownPool;
+      }
+    }
+
+    const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
+    const address = txMinerInfo.vout[0].scriptpubkey_address;
+
+    let pools: PoolTag[] = [];
+    if (config.DATABASE.ENABLED === true) {
+      pools = await poolsRepository.$getPools();
+    } else {
+      pools = poolsParser.miningPools;
+    }
+    for (let i = 0; i < pools.length; ++i) {
+      if (address !== undefined) {
+        const addresses: string[] = JSON.parse(pools[i].addresses);
+        if (addresses.indexOf(address) !== -1) {
+          return pools[i];
+        }
+      }
+
+      const regexes: string[] = JSON.parse(pools[i].regexes);
+      for (let y = 0; y < regexes.length; ++y) {
+        const regex = new RegExp(regexes[y], 'i');
+        const match = asciiScriptSig.match(regex);
+        if (match !== null) {
+          return pools[i];
+        }
+      }
+    }
+
+    if (config.DATABASE.ENABLED === true) {
+      return await poolsRepository.$getUnknownPool();
+    } else {
+      return poolsParser.unknownPool;
+    }
+  }
+
+  /**
+   * [INDEXING] Index all blocks summaries for the block txs visualization
+   */
+  public async $generateBlocksSummariesDatabase() {
+    if (Common.blocksSummariesIndexingEnabled() === false) {
+      return;
+    }
+
+    try {
+      // Get all indexed block hash
+      const indexedBlocks = await blocksRepository.$getIndexedBlocks();
+      const indexedBlockSummariesHashesArray = await BlocksSummariesRepository.$getIndexedSummariesId();
+
+      const indexedBlockSummariesHashes = {}; // Use a map for faster seek during the indexing loop
+      for (const hash of indexedBlockSummariesHashesArray) {
+        indexedBlockSummariesHashes[hash] = true;
+      }
+
+      // Logging
+      let newlyIndexed = 0;
+      let totalIndexed = indexedBlockSummariesHashesArray.length;
+      let indexedThisRun = 0;
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+
+      for (const block of indexedBlocks) {
+        if (indexedBlockSummariesHashes[block.hash] === true) {
+          continue;
+        }
+
+        // Logging
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 5) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
+          const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
+          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+        }
+
+        await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
+
+        // Logging
+        indexedThisRun++;
+        totalIndexed++;
+        newlyIndexed++;
+      }
+      if (newlyIndexed > 0) {
+        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+      } else {
+        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+      }
+    } catch (e) {
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Index all blocks metadata for the mining dashboard
+   */
+  public async $generateBlockDatabase(): Promise<boolean> {
+    try {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      let currentBlockHeight = blockchainInfo.blocks;
+
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+
+      const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
+      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
+      loadingIndicators.setProgress('block-indexing', 0);
+
+      const chunkSize = 10000;
+      let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
+      let indexedThisRun = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      while (currentBlockHeight >= lastBlockToIndex) {
+        const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
+
+        const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
+          currentBlockHeight, endBlock);
+        if (missingBlockHeights.length <= 0) {
+          currentBlockHeight -= chunkSize;
+          continue;
+        }
+
+        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
+
+        for (const blockHeight of missingBlockHeights) {
+          if (blockHeight < lastBlockToIndex) {
+            break;
+          }
+          ++indexedThisRun;
+          ++totalIndexed;
+          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+          if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
+            const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+            const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
+            const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`);
+            timer = new Date().getTime() / 1000;
+            indexedThisRun = 0;
+            loadingIndicators.setProgress('block-indexing', progress, false);
+          }
+          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
+          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
+          const blockExtended = await this.$getBlockExtended(block, transactions);
+
+          newlyIndexed++;
+          await blocksRepository.$saveBlockInDatabase(blockExtended);
+        }
+
+        currentBlockHeight -= chunkSize;
+      }
+      if (newlyIndexed > 0) {
+        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+      } else {
+        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+      }
+      loadingIndicators.setProgress('block-indexing', 100);
+    } catch (e) {
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
+      loadingIndicators.setProgress('block-indexing', 100);
+      throw e;
+    }
+
+    return await BlocksRepository.$validateChain();
+  }
+
   public async $updateBlocks() {
+    let fastForwarded = false;
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
 
     if (this.blocks.length === 0) {
-      this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT;
+      this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1);
     } else {
       this.currentBlockHeight = this.blocks[this.blocks.length - 1].height;
     }
@@ -42,6 +395,9 @@ class Blocks {
     if (blockHeightTip - this.currentBlockHeight > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 2) {
       logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`);
       this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT;
+      fastForwarded = true;
+      logger.info(`Re-indexing skipped blocks and corresponding hashrates data`);
+      indexer.reindex(); // Make sure to index the skipped blocks #1619
     }
 
     if (!this.lastDifficultyAdjustmentTime) {
@@ -49,71 +405,74 @@ class Blocks {
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
-        const block = await bitcoinApi.$getBlock(blockHash);
+        const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
-        const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-        const previousPeriodBlock = await bitcoinApi.$getBlock(previousPeriodBlockHash);
-        this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
-        logger.debug(`Initial difficulty adjustment data set.`);
+        if (blockHeightTip >= 2016) {
+          const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
+          const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
+          this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
+          logger.debug(`Initial difficulty adjustment data set.`);
+        }
       } else {
         logger.debug(`Blockchain headers (${blockchainInfo.headers}) and blocks (${blockchainInfo.blocks}) not in sync. Waiting...`);
       }
     }
 
     while (this.currentBlockHeight < blockHeightTip) {
-      if (this.currentBlockHeight === 0) {
+      if (this.currentBlockHeight < blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT) {
         this.currentBlockHeight = blockHeightTip;
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
       }
 
-      const transactions: TransactionExtended[] = [];
-
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
-      const block = await bitcoinApi.$getBlock(blockHash);
+      const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
+      const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
+      const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+      const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
 
-      const mempool = memPool.getMempool();
-      let transactionsFound = 0;
-
-      for (let i = 0; i < txIds.length; i++) {
-        if (mempool[txIds[i]]) {
-          transactions.push(mempool[txIds[i]]);
-          transactionsFound++;
-        } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
-          logger.debug(`Fetching block tx ${i} of ${txIds.length}`);
-          try {
-            const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
-            transactions.push(tx);
-          } catch (e) {
-            logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
-            if (i === 0) {
-              throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
+      if (Common.indexingEnabled()) {
+        if (!fastForwarded) {
+          const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
+          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
+            logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
+            // We assume there won't be a reorg with more than 10 block depth
+            await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await HashratesRepository.$deleteLastEntries();
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            for (let i = 10; i >= 0; --i) {
+              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
+              await this.$getStrippedBlockTransactions(newBlock.id, true, true);
             }
+            await mining.$indexDifficultyAdjustments();
+            await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            indexer.reindex();
+          }
+          await blocksRepository.$saveBlockInDatabase(blockExtended);
+
+          // Save blocks summary for visualization if it's enabled
+          if (Common.blocksSummariesIndexingEnabled() === true) {
+            await this.$getStrippedBlockTransactions(blockExtended.id, true);
           }
         }
       }
 
-      transactions.forEach((tx) => {
-        if (!tx.cpfpChecked) {
-          Common.setRelativesAndGetCpfpInfo(tx, mempool);
-        }
-      });
-
-      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`);
-
-      const blockExtended: BlockExtended = Object.assign({}, block);
-      blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-      blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-      transactions.shift();
-      transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
-      blockExtended.medianFee = transactions.length > 0 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0;
-      blockExtended.feeRange = transactions.length > 0 ? Common.getFeesInRange(transactions, 8) : [0, 0];
-
       if (block.height % 2016 === 0) {
+        if (Common.indexingEnabled()) {
+          await DifficultyAdjustmentsRepository.$saveAdjustments({
+            time: block.timestamp,
+            height: block.height,
+            difficulty: block.difficulty,
+            adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
+          });
+        }
+
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
@@ -123,16 +482,144 @@ class Blocks {
       if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
         this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
       }
+      this.blockSummaries.push(blockSummary);
+      if (this.blockSummaries.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
+        this.blockSummaries = this.blockSummaries.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
+      }
 
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (memPool.isInSync()) {
+      if (!memPool.hasPriority()) {
         diskCache.$saveCacheToDisk();
       }
     }
   }
 
+  /**
+   * Index a block if it's missing from the database. Returns the block after indexing
+   */
+  public async $indexBlock(height: number): Promise<BlockExtended> {
+    const dbBlock = await blocksRepository.$getBlockByHeight(height);
+    if (dbBlock != null) {
+      return prepareBlock(dbBlock);
+    }
+
+    const blockHash = await bitcoinApi.$getBlockHash(height);
+    const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+    const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+    const blockExtended = await this.$getBlockExtended(block, transactions);
+
+    await blocksRepository.$saveBlockInDatabase(blockExtended);
+
+    return prepareBlock(blockExtended);
+  }
+
+  /**
+   * Index a block by hash if it's missing from the database. Returns the block after indexing
+   */
+  public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
+    // Check the memory cache
+    const blockByHash = this.getBlocks().find((b) => b.id === hash);
+    if (blockByHash) {
+      return blockByHash;
+    }
+
+    // Block has already been indexed
+    if (Common.indexingEnabled()) {
+      const dbBlock = await blocksRepository.$getBlockByHash(hash);
+      if (dbBlock != null) {
+        return prepareBlock(dbBlock);
+      }
+    }
+
+    let block = await bitcoinClient.getBlock(hash);
+
+    // Not Bitcoin network, return the block as it
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return block;
+    }
+
+    block = prepareBlock(block);
+
+    // Bitcoin network, add our custom data on top
+    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
+    const blockExtended = await this.$getBlockExtended(block, transactions);
+    if (Common.indexingEnabled()) {
+      delete(blockExtended['coinbaseTx']);
+      await blocksRepository.$saveBlockInDatabase(blockExtended);
+    }
+
+    return blockExtended;
+  }
+
+  public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false,
+    skipDBLookup: boolean = false): Promise<TransactionStripped[]>
+  {
+    if (skipMemoryCache === false) {
+      // Check the memory cache
+      const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);
+      if (cachedSummary) {
+        return cachedSummary.transactions;
+      }
+    }
+
+    // Check if it's indexed in db
+    if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) {
+      const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash);
+      if (indexedSummary !== undefined) {
+        return indexedSummary.transactions;
+      }
+    }
+
+    // Call Core RPC
+    const block = await bitcoinClient.getBlock(hash, 2);
+    const summary = this.summarizeBlock(block);
+
+    // Index the response if needed
+    if (Common.blocksSummariesIndexingEnabled() === true) {
+      await BlocksSummariesRepository.$saveSummary(block.height, summary);
+    }
+
+    return summary.transactions;
+  }
+
+  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
+    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+    const returnBlocks: BlockExtended[] = [];
+
+    if (currentHeight < 0) {
+      return returnBlocks;
+    }
+
+    // Check if block height exist in local cache to skip the hash lookup
+    const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
+    let startFromHash: string | null = null;
+    if (blockByHeight) {
+      startFromHash = blockByHeight.id;
+    } else if (!Common.indexingEnabled()) {
+      startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
+    }
+
+    let nextHash = startFromHash;
+    for (let i = 0; i < limit && currentHeight >= 0; i++) {
+      let block = this.getBlocks().find((b) => b.height === currentHeight);
+      if (block) {
+        returnBlocks.push(block);
+      } else if (Common.indexingEnabled()) {
+        block = await this.$indexBlock(currentHeight);
+        returnBlocks.push(block);
+      } else if (nextHash != null) {
+        block = prepareBlock(await bitcoinClient.getBlock(nextHash));
+        nextHash = block.previousblockhash;
+        returnBlocks.push(block);
+      }
+      currentHeight--;
+    }
+
+    return returnBlocks;
+  }
+
   public getLastDifficultyAdjustmentTime(): number {
     return this.lastDifficultyAdjustmentTime;
   }

backend/src/api/common.ts

@@ -1,7 +1,14 @@
 import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
 export class Common {
-  static nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
+  static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
+    '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
+  : '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
+  static _isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
+
+  static isLiquid(): boolean {
+    return this._isLiquid;
+  }
 
   static median(numbers: number[]) {
     let medianNr = 0;
@@ -70,7 +77,7 @@ export class Common {
     };
   }
 
-  static sleep(ms: number): Promise<void> {
+  static sleep$(ms: number): Promise<void> {
     return new Promise((resolve) => {
        setTimeout(() => {
          resolve();
@@ -107,7 +114,7 @@ export class Common {
       totalFees += tx.bestDescendant.fee;
     }
 
-    tx.effectiveFeePerVsize = Math.max(config.MEMPOOL.NETWORK === 'liquid' ? 0.1 : 1, totalFees / (totalWeight / 4));
+    tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4));
     tx.cpfpChecked = true;
 
     return {
@@ -147,4 +154,34 @@ export class Common {
     });
     return parents;
   }
+
+  static getSqlInterval(interval: string | null): string | null {
+    switch (interval) {
+      case '24h': return '1 DAY';
+      case '3d': return '3 DAY';
+      case '1w': return '1 WEEK';
+      case '1m': return '1 MONTH';
+      case '3m': return '3 MONTH';
+      case '6m': return '6 MONTH';
+      case '1y': return '1 YEAR';
+      case '2y': return '2 YEAR';
+      case '3y': return '3 YEAR';
+      default: return null;
+    }
+  }
+
+  static indexingEnabled(): boolean {
+    return (
+      ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
+      config.DATABASE.ENABLED === true &&
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
+    );
+  }
+
+  static blocksSummariesIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
+    );
+  }
 }

backend/src/api/database-migration.ts

@@ -0,0 +1,610 @@
+import config from '../config';
+import DB from '../database';
+import logger from '../logger';
+import { Common } from './common';
+
+class DatabaseMigration {
+  private static currentVersion = 24;
+  private queryTimeout = 120000;
+  private statisticsAddedIndexed = false;
+  private uniqueLogs: string[] = [];
+
+  private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`;
+  private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`;
+
+  constructor() { }
+
+  /**
+   * Avoid printing multiple time the same message
+   */
+  private uniqueLog(loggerFunction: any, msg: string) {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
+  /**
+   * Entry point
+   */
+  public async $initializeOrMigrateDatabase(): Promise<void> {
+    logger.debug('MIGRATIONS: Running migrations');
+
+    await this.$printDatabaseVersion();
+
+    // First of all, if the `state` database does not exist, create it so we can track migration version
+    if (!await this.$checkIfTableExists('state')) {
+      logger.debug('MIGRATIONS: `state` table does not exist. Creating it.');
+      try {
+        await this.$createMigrationStateTable();
+      } catch (e) {
+        logger.err('MIGRATIONS: Unable to create `state` table, aborting in 10 seconds. ' + e);
+        await Common.sleep$(10000);
+        process.exit(-1);
+      }
+      logger.debug('MIGRATIONS: `state` table initialized.');
+    }
+
+    let databaseSchemaVersion = 0;
+    try {
+      databaseSchemaVersion = await this.$getSchemaVersionFromDatabase();
+    } catch (e) {
+      logger.err('MIGRATIONS: Unable to get current database migration version, aborting in 10 seconds. ' + e);
+      await Common.sleep$(10000);
+      process.exit(-1);
+    }
+
+    if (databaseSchemaVersion === 0) {
+      logger.info('Initializing database (first run, clean install)');
+    }
+
+    if (databaseSchemaVersion <= 2) {
+      // Disable some spam logs when they're not relevant
+      this.uniqueLogs.push(this.blocksTruncatedMessage);
+      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+    }
+
+    logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
+    logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
+    if (databaseSchemaVersion >= DatabaseMigration.currentVersion) {
+      logger.debug('MIGRATIONS: Nothing to do.');
+      return;
+    }
+
+    // Now, create missing tables. Those queries cannot be wrapped into a transaction unfortunately
+    try {
+      await this.$createMissingTablesAndIndexes(databaseSchemaVersion);
+    } catch (e) {
+      logger.err('MIGRATIONS: Unable to create required tables, aborting in 10 seconds. ' + e);
+      await Common.sleep$(10000);
+      process.exit(-1);
+    }
+
+    if (DatabaseMigration.currentVersion > databaseSchemaVersion) {
+      try {
+        await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
+        if (databaseSchemaVersion === 0) {
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
+        } else {
+          logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
+        }
+      } catch (e) {
+        logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e);
+      }
+    }
+
+    return;
+  }
+
+  /**
+   * Create all missing tables
+   */
+  private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
+    await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
+
+    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
+    try {
+      await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
+      await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
+      if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
+        await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
+      }
+      if (databaseSchemaVersion < 3) {
+        await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+      }
+      if (databaseSchemaVersion < 4) {
+        await this.$executeQuery('DROP table IF EXISTS blocks;');
+        await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
+      }
+      if (databaseSchemaVersion < 5 && isBitcoin === true) {
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 6 && isBitcoin === true) {
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+        await this.$executeQuery('TRUNCATE blocks;');  // Need to re-index
+        // Cleanup original blocks fields type
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
+        // We also fix the pools.id type so we need to drop/re-create the foreign key
+        await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
+        await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
+        await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
+        // Add new block indexing fields
+        await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
+        await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
+      }
+
+      if (databaseSchemaVersion < 7 && isBitcoin === true) {
+        await this.$executeQuery('DROP table IF EXISTS hashrates;');
+        await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
+      }
+
+      if (databaseSchemaVersion < 8 && isBitcoin === true) {
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
+      }
+
+      if (databaseSchemaVersion < 9 && isBitcoin === true) {
+        this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
+      }
+
+      if (databaseSchemaVersion < 10 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
+      }
+
+      if (databaseSchemaVersion < 11 && isBitcoin === true) {
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+        await this.$executeQuery(`ALTER TABLE blocks
+          ADD avg_fee INT UNSIGNED NULL,
+          ADD avg_fee_rate INT UNSIGNED NULL
+        `);
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 12 && isBitcoin === true) {
+        // No need to re-index because the new data type can contain larger values
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 13 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 14 && isBitcoin === true) {
+        this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
+        await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 16 && isBitcoin === true) {
+        this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
+      }
+
+      if (databaseSchemaVersion < 17 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
+      }
+
+      if (databaseSchemaVersion < 18 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
+      }
+
+      if (databaseSchemaVersion < 19) {
+        await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
+      }
+
+      if (databaseSchemaVersion < 20 && isBitcoin === true) {
+        await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
+      }
+
+      if (databaseSchemaVersion < 21) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
+        await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
+      }
+
+      if (databaseSchemaVersion < 22 && isBitcoin === true) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
+        await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
+      }
+
+      if (databaseSchemaVersion < 23) {
+        await this.$executeQuery('TRUNCATE `prices`');
+        await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 24 && isBitcoin == true) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
+        await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
+      }
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  /**
+   * Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed
+   * while it does not appear in previous schemas. The mariadb command "CREATE INDEX IF NOT EXISTS" is not supported on
+   * older mariadb version. Therefore we set a flag here in order to know if the index needs to be created or not before
+   * running the migration process
+   */
+  private async $setStatisticsAddedIndexedFlag(databaseSchemaVersion: number) {
+    if (databaseSchemaVersion >= 2) {
+      this.statisticsAddedIndexed = true;
+      return;
+    }
+
+    try {
+      // We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X
+      const query = `SELECT COUNT(1) hasIndex FROM INFORMATION_SCHEMA.STATISTICS
+        WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`;
+      const [rows] = await this.$executeQuery(query, true);
+      if (rows[0].hasIndex === 0) {
+        logger.debug('MIGRATIONS: `statistics.added` is not indexed');
+        this.statisticsAddedIndexed = false;
+      } else if (rows[0].hasIndex === 1) {
+        logger.debug('MIGRATIONS: `statistics.added` is already indexed');
+        this.statisticsAddedIndexed = true;
+      }
+    } catch (e) {
+      // Should really never happen but just in case it fails, we just don't execute
+      // any query related to this indexing so it won't fail if the index actually already exists
+      logger.err('MIGRATIONS: Unable to check if `statistics.added` INDEX exist or not.');
+      this.statisticsAddedIndexed = true;
+    }
+  }
+
+  /**
+   * Small query execution wrapper to log all executed queries
+   */
+  private async $executeQuery(query: string, silent: boolean = false): Promise<any> {
+    if (!silent) {
+      logger.debug('MIGRATIONS: Execute query:\n' + query);
+    }
+    return DB.query({ sql: query, timeout: this.queryTimeout });
+  }
+
+  /**
+   * Check if 'table' exists in the database
+   */
+  private async $checkIfTableExists(table: string): Promise<boolean> {
+    const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
+    const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+    return rows[0]['COUNT(*)'] === 1;
+  }
+
+  /**
+   * Get current database version
+   */
+  private async $getSchemaVersionFromDatabase(): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = 'schema_version';`;
+    const [rows] = await this.$executeQuery(query, true);
+    return rows[0]['number'];
+  }
+
+  /**
+   * Create the `state` table
+   */
+  private async $createMigrationStateTable(): Promise<void> {
+    try {
+      const query = `CREATE TABLE IF NOT EXISTS state (
+        name varchar(25) NOT NULL,
+        number int(11) NULL,
+        string varchar(100) NULL,
+        CONSTRAINT name_unique UNIQUE (name)
+      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+      await this.$executeQuery(query);
+
+      // Set initial values
+      await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
+      await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  /**
+   * We actually execute the migrations queries here
+   */
+  private async $migrateTableSchemaFromVersion(version: number): Promise<void> {
+    const transactionQueries: string[] = [];
+    for (const query of this.getMigrationQueriesFromVersion(version)) {
+      transactionQueries.push(query);
+    }
+
+    logger.notice(`MIGRATIONS: ${version > 0 ? 'Upgrading' : 'Initializing'} database schema version number to ${DatabaseMigration.currentVersion}`);
+    transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
+
+    try {
+      await this.$executeQuery('START TRANSACTION;');
+      for (const query of transactionQueries) {
+        await this.$executeQuery(query);
+      }
+      await this.$executeQuery('COMMIT;');
+    } catch (e) {
+      await this.$executeQuery('ROLLBACK;');
+      throw e;
+    }
+  }
+
+  /**
+   * Generate migration queries based on schema version
+   */
+  private getMigrationQueriesFromVersion(version: number): string[] {
+    const queries: string[] = [];
+    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
+
+    if (version < 1) {
+      if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
+        if (version > 0) {
+          logger.notice(`MIGRATIONS: Migrating (shifting) statistics table data`);
+        }
+        queries.push(this.getShiftStatisticsQuery());
+      }
+    }
+
+    if (version < 7 && isBitcoin === true) {
+      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
+    }
+
+    if (version < 9  && isBitcoin === true) {
+      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
+    }
+
+    return queries;
+  }
+
+  /**
+   * Save the schema version in the database
+   */
+  private getUpdateToLatestSchemaVersionQuery(): string {
+    return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
+  }
+
+  /**
+   * Print current database version
+   */
+  private async $printDatabaseVersion() {
+    try {
+      const [rows] = await this.$executeQuery('SELECT VERSION() as version;', true);
+      logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`);
+    } catch (e) {
+      logger.debug(`MIGRATIONS: Could not fetch database engine version. ` + e);
+    }
+  }
+
+  // Couple of wrappers to clean the main logic
+  private getShiftStatisticsQuery(): string {
+    return `UPDATE statistics SET
+      vsize_1 = vsize_1 + vsize_2, vsize_2 = vsize_3,
+      vsize_3 = vsize_4, vsize_4 = vsize_5,
+      vsize_5 = vsize_6, vsize_6 = vsize_8,
+      vsize_8 = vsize_10, vsize_10 = vsize_12,
+      vsize_12 = vsize_15, vsize_15 = vsize_20,
+      vsize_20 = vsize_30, vsize_30 = vsize_40,
+      vsize_40 = vsize_50, vsize_50 = vsize_60,
+      vsize_60 = vsize_70, vsize_70 = vsize_80,
+      vsize_80 = vsize_90, vsize_90 = vsize_100,
+      vsize_100 = vsize_125, vsize_125 = vsize_150,
+      vsize_150 = vsize_175, vsize_175 = vsize_200,
+      vsize_200 = vsize_250, vsize_250 = vsize_300,
+      vsize_300 = vsize_350, vsize_350 = vsize_400,
+      vsize_400 = vsize_500, vsize_500 = vsize_600,
+      vsize_600 = vsize_700, vsize_700 = vsize_800,
+      vsize_800 = vsize_900, vsize_900 = vsize_1000,
+      vsize_1000 = vsize_1200, vsize_1200 = vsize_1400,
+      vsize_1400 = vsize_1800, vsize_1800 = vsize_2000, vsize_2000 = 0;`;
+  }
+
+  private getCreateStatisticsQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS statistics (
+      id int(11) NOT NULL AUTO_INCREMENT,
+      added datetime NOT NULL,
+      unconfirmed_transactions int(11) UNSIGNED NOT NULL,
+      tx_per_second float UNSIGNED NOT NULL,
+      vbytes_per_second int(10) UNSIGNED NOT NULL,
+      mempool_byte_weight int(10) UNSIGNED NOT NULL,
+      fee_data longtext NOT NULL,
+      total_fee double UNSIGNED NOT NULL,
+      vsize_1 int(11) NOT NULL,
+      vsize_2 int(11) NOT NULL,
+      vsize_3 int(11) NOT NULL,
+      vsize_4 int(11) NOT NULL,
+      vsize_5 int(11) NOT NULL,
+      vsize_6 int(11) NOT NULL,
+      vsize_8 int(11) NOT NULL,
+      vsize_10 int(11) NOT NULL,
+      vsize_12 int(11) NOT NULL,
+      vsize_15 int(11) NOT NULL,
+      vsize_20 int(11) NOT NULL,
+      vsize_30 int(11) NOT NULL,
+      vsize_40 int(11) NOT NULL,
+      vsize_50 int(11) NOT NULL,
+      vsize_60 int(11) NOT NULL,
+      vsize_70 int(11) NOT NULL,
+      vsize_80 int(11) NOT NULL,
+      vsize_90 int(11) NOT NULL,
+      vsize_100 int(11) NOT NULL,
+      vsize_125 int(11) NOT NULL,
+      vsize_150 int(11) NOT NULL,
+      vsize_175 int(11) NOT NULL,
+      vsize_200 int(11) NOT NULL,
+      vsize_250 int(11) NOT NULL,
+      vsize_300 int(11) NOT NULL,
+      vsize_350 int(11) NOT NULL,
+      vsize_400 int(11) NOT NULL,
+      vsize_500 int(11) NOT NULL,
+      vsize_600 int(11) NOT NULL,
+      vsize_700 int(11) NOT NULL,
+      vsize_800 int(11) NOT NULL,
+      vsize_900 int(11) NOT NULL,
+      vsize_1000 int(11) NOT NULL,
+      vsize_1200 int(11) NOT NULL,
+      vsize_1400 int(11) NOT NULL,
+      vsize_1600 int(11) NOT NULL,
+      vsize_1800 int(11) NOT NULL,
+      vsize_2000 int(11) NOT NULL,
+      CONSTRAINT PRIMARY KEY (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateElementsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS elements_pegs (
+      block int(11) NOT NULL,
+      datetime int(11) NOT NULL,
+      amount bigint(20) NOT NULL,
+      txid varchar(65) NOT NULL,
+      txindex int(11) NOT NULL,
+      bitcoinaddress varchar(100) NOT NULL,
+      bitcointxid varchar(65) NOT NULL,
+      bitcoinindex int(11) NOT NULL,
+      final_tx int(11) NOT NULL
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreatePoolsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS pools (
+      id int(11) NOT NULL AUTO_INCREMENT,
+      name varchar(50) NOT NULL,
+      link varchar(255) NOT NULL,
+      addresses text NOT NULL,
+      regexes text NOT NULL,
+      PRIMARY KEY (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`;
+  }
+
+  private getCreateBlocksTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks (
+      height int(11) unsigned NOT NULL,
+      hash varchar(65) NOT NULL,
+      blockTimestamp timestamp NOT NULL,
+      size int(11) unsigned NOT NULL,
+      weight int(11) unsigned NOT NULL,
+      tx_count int(11) unsigned NOT NULL,
+      coinbase_raw text,
+      difficulty bigint(20) unsigned NOT NULL,
+      pool_id int(11) DEFAULT -1,
+      fees double unsigned NOT NULL,
+      fee_span json NOT NULL,
+      median_fee double unsigned NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (pool_id),
+      FOREIGN KEY (pool_id) REFERENCES pools (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateDailyStatsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS hashrates (
+      hashrate_timestamp timestamp NOT NULL,
+      avg_hashrate double unsigned DEFAULT '0',
+      pool_id smallint unsigned NULL,
+      PRIMARY KEY (hashrate_timestamp),
+      INDEX (pool_id),
+      FOREIGN KEY (pool_id) REFERENCES pools (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table
+    return `CREATE TABLE IF NOT EXISTS rates (
+      height int(10) unsigned NOT NULL,
+      bisq_rates JSON NOT NULL,
+      PRIMARY KEY (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateBlocksSummariesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_summaries (
+      height int(10) unsigned NOT NULL,
+      id varchar(65) NOT NULL,
+      transactions JSON NOT NULL,
+      PRIMARY KEY (id),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreatePricesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS prices (
+      time timestamp NOT NULL,
+      avg_prices JSON NOT NULL,
+      PRIMARY KEY (time)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateDifficultyAdjustmentsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS difficulty_adjustments (
+      time timestamp NOT NULL,
+      height int(10) unsigned NOT NULL,
+      difficulty double unsigned NOT NULL,
+      adjustment float NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (time)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateBlocksAuditsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_audits (
+      time timestamp NOT NULL,
+      hash varchar(65) NOT NULL,
+      height int(10) unsigned NOT NULL,
+      missing_txs JSON NOT NULL,
+      added_txs JSON NOT NULL,
+      match_rate float unsigned NOT NULL,
+      PRIMARY KEY (hash),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  public async $truncateIndexedData(tables: string[]) {
+    const allowedTables = ['blocks', 'hashrates', 'prices'];
+
+    try {
+      for (const table of tables) {
+        if (!allowedTables.includes(table)) {
+          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
+          continue;
+        }
+
+        await this.$executeQuery(`TRUNCATE ${table}`, true);
+        if (table === 'hashrates') {
+          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
+        }
+        logger.notice(`Table ${table} has been truncated`);
+      }
+    } catch (e) {
+      logger.warn(`Unable to erase indexed data`);
+    }
+  }
+}
+
+export default new DatabaseMigration();

backend/src/api/difficulty-adjustment.ts

@@ -0,0 +1,67 @@
+import config from '../config';
+import { IDifficultyAdjustment } from '../mempool.interfaces';
+import blocks from './blocks';
+
+class DifficultyAdjustmentApi {
+  constructor() { }
+
+  public getDifficultyAdjustment(): IDifficultyAdjustment {
+    const DATime = blocks.getLastDifficultyAdjustmentTime();
+    const previousRetarget = blocks.getPreviousDifficultyRetarget();
+    const blockHeight = blocks.getCurrentBlockHeight();
+    const blocksCache = blocks.getBlocks();
+    const latestBlock = blocksCache[blocksCache.length - 1];
+
+    const now = new Date().getTime() / 1000;
+    const diff = now - DATime;
+    const blocksInEpoch = blockHeight % 2016;
+    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
+    const remainingBlocks = 2016 - blocksInEpoch;
+    const nextRetargetHeight = blockHeight + remainingBlocks;
+
+    let difficultyChange = 0;
+    if (remainingBlocks < 1870) {
+      if (blocksInEpoch > 0) {
+        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
+      }
+      if (difficultyChange > 300) {
+        difficultyChange = 300;
+      }
+      if (difficultyChange < -75) {
+        difficultyChange = -75;
+      }
+    }
+
+    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
+
+    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+    // therefore the time between blocks will always be below 20 minutes (1200s).
+    let timeOffset = 0;
+    if (config.MEMPOOL.NETWORK === 'testnet') {
+      if (timeAvgMins > 20) {
+        timeAvgMins = 20;
+      }
+      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
+        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
+      }
+    }
+
+    const timeAvg = timeAvgMins * 60 * 1000 ;
+    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
+    const estimatedRetargetDate = remainingTime + now;
+
+    return {
+      progressPercent,
+      difficultyChange,
+      estimatedRetargetDate,
+      remainingBlocks,
+      remainingTime,
+      previousRetarget,
+      nextRetargetHeight,
+      timeAvg,
+      timeOffset,
+    };
+  }
+}
+
+export default new DifficultyAdjustmentApi();

backend/src/api/disk-cache.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 const fsPromises = fs.promises;
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
@@ -9,6 +9,8 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
+  private cacheSchemaVersion = 1;
+
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
   private static CHUNK_FILES = 25;
@@ -17,7 +19,7 @@ class DiskCache {
   constructor() { }
 
   async $saveCacheToDisk(): Promise<void> {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
       return;
     }
     if (this.isWritingCache) {
@@ -39,15 +41,17 @@ class DiskCache {
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
       await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
+        cacheSchemaVersion: this.cacheSchemaVersion,
         blocks: blocks.getBlocks(),
+        blockSummaries: blocks.getBlockSummaries(),
         mempool: {},
         mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), {flag: 'w'});
+      }), { flag: 'w' });
       for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
         await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), {flag: 'w'});
+        }), { flag: 'w' });
       }
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
@@ -57,6 +61,13 @@ class DiskCache {
     }
   }
 
+  wipeCache() {
+    fs.unlinkSync(DiskCache.FILE_NAME);
+    for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+      fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+    }
+  }
+
   loadMempoolCache() {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
@@ -67,6 +78,11 @@ class DiskCache {
       if (cacheData) {
         logger.info('Restoring mempool and blocks data from disk cache');
         data = JSON.parse(cacheData);
+        if (data.cacheSchemaVersion === undefined || data.cacheSchemaVersion !== this.cacheSchemaVersion) {
+          logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
+          return this.wipeCache();
+        }
+
         if (data.mempoolArray) {
           for (const tx of data.mempoolArray) {
             data.mempool[tx.txid] = tx;
@@ -88,14 +104,15 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.debug('Error parsing ' + fileName + '. Skipping.');
+          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
         }
       }
 
       memPool.setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
+      blocks.setBlockSummaries(data.blockSummaries || []);
     } catch (e) {
-      logger.warn('Failed to parse mempoool and blocks cache. Skipping.');
+      logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/api/fee-api.ts

@@ -1,12 +1,12 @@
-import config from '../config';
 import { MempoolBlock } from '../mempool.interfaces';
+import { Common } from './common';
 import mempool from './mempool';
 import projectedBlocks from './mempool-blocks';
 
 class FeeApi {
   constructor() { }
 
-  defaultFee = config.MEMPOOL.NETWORK === 'liquid' ? 0.1 : 1;
+  defaultFee = Common.isLiquid() ? 0.1 : 1;
 
   public getRecommendedFee() {
     const pBlocks = projectedBlocks.getMempoolBlocks();
@@ -18,6 +18,7 @@ class FeeApi {
         'fastestFee': this.defaultFee,
         'halfHourFee': this.defaultFee,
         'hourFee': this.defaultFee,
+        'economyFee': minimumFee,
         'minimumFee': minimumFee,
       };
     }
@@ -30,6 +31,7 @@ class FeeApi {
       'fastestFee': firstMedianFee,
       'halfHourFee': secondMedianFee,
       'hourFee': thirdMedianFee,
+      'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
       'minimumFee': minimumFee,
     };
   }

backend/src/api/fiat-conversion.ts

@@ -1,22 +1,39 @@
 import logger from '../logger';
-import axios from 'axios';
+import * as http from 'http';
+import * as https from 'https';
+import axios, { AxiosResponse } from 'axios';
 import { IConversionRates } from '../mempool.interfaces';
 import config from '../config';
+import backendInfo from './backend-info';
+import { SocksProxyAgent } from 'socks-proxy-agent';
 
 class FiatConversion {
-  private conversionRates: IConversionRates = {
-    'USD': 0
-  };
+  private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
+    'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
+    'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
+    'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
+  private conversionRates: IConversionRates = {};
   private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
+  public ratesInitialized = false; // If true, it means rates are ready for use
 
-  constructor() { }
+  constructor() {
+    for (const fiat of this.debasingFiatCurrencies) {
+      this.conversionRates[fiat] = 0;
+    }
+  }
 
   public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
     this.ratesChangedCallback = fn;
   }
 
   public startService() {
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
     logger.info('Starting currency rates service');
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
+    } else {
+      logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
+    }
     setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
     this.updateCurrency();
   }
@@ -26,17 +43,79 @@ class FiatConversion {
   }
 
   private async updateCurrency(): Promise<void> {
-    try {
-      const response = await axios.get('https://price.bisq.wiz.biz/getAllMarketPrices', { timeout: 10000 });
-      const usd = response.data.data.find((item: any) => item.currencyCode === 'USD');
-      this.conversionRates = {
-        'USD': usd.price,
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
       };
-      if (this.ratesChangedCallback) {
-        this.ratesChangedCallback(this.conversionRates);
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
+    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+
+    let retry = 0;
+
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          let socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        logger.debug('Querying currency rates service...');
+
+        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
+
+        if (response.statusText === 'error' || !response.data) {
+          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
+        }
+
+        for (const rate of response.data.data) {
+          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
+            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
+          }
+        }
+
+        this.ratesInitialized = true;
+        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
+
+        if (this.ratesChangedCallback) {
+          this.ratesChangedCallback(this.conversionRates);
+        }
+        break;
+      } catch (e) {
+        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
       }
-    } catch (e) {
-      logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/api/liquid/elements-parser.ts

@@ -2,7 +2,7 @@ import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface';
 import bitcoinClient from '../bitcoin/bitcoin-client';
 import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
 import { Common } from '../common';
-import { DB } from '../../database';
+import DB from '../../database';
 import logger from '../../logger';
 
 class ElementsParser {
@@ -18,12 +18,12 @@ class ElementsParser {
       this.isRunning = true;
       const result = await bitcoinClient.getChainTips();
       const tip = result[0].height;
-      const latestBlock = await this.$getLatestBlockFromDatabase();
-      for (let height = latestBlock.block + 1; height <= tip; height++) {
+      const latestBlockHeight = await this.$getLatestBlockHeightFromDatabase();
+      for (let height = latestBlockHeight + 1; height <= tip; height++) {
         const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height);
         const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2);
         await this.$parseBlock(block);
-        await this.$saveLatestBlockToDatabase(block.height, block.time, block.hash);
+        await this.$saveLatestBlockToDatabase(block.height);
       }
       this.isRunning = false;
     } catch (e) {
@@ -33,10 +33,8 @@ class ElementsParser {
   }
 
   public async $getPegDataByMonth(): Promise<any> {
-    const connection = await DB.pool.getConnection();
     const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
-    const [rows] = await connection.query<any>(query);
-    connection.release();
+    const [rows] = await DB.query(query);
     return rows;
   }
 
@@ -79,7 +77,6 @@ class ElementsParser {
 
   protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
     txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
-    const connection = await DB.pool.getConnection();
     const query = `INSERT INTO elements_pegs(
         block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -87,24 +84,19 @@ class ElementsParser {
     const params: (string | number)[] = [
       height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
     ];
-    await connection.query(query, params);
-    connection.release();
+    await DB.query(query, params);
     logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
   }
 
-  protected async $getLatestBlockFromDatabase(): Promise<any> {
-    const connection = await DB.pool.getConnection();
-    const query = `SELECT block, datetime, block_hash FROM last_elements_block`;
-    const [rows] = await connection.query<any>(query);
-    connection.release();
-    return rows[0];
+  protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
+    const [rows] = await DB.query(query);
+    return rows[0]['number'];
   }
 
-  protected async $saveLatestBlockToDatabase(blockHeight: number, datetime: number, blockHash: string) {
-    const connection = await DB.pool.getConnection();
-    const query = `UPDATE last_elements_block SET block = ?, datetime = ?, block_hash = ?`;
-    await connection.query<any>(query, [blockHeight, datetime, blockHash]);
-    connection.release();
+  protected async $saveLatestBlockToDatabase(blockHeight: number) {
+    const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
+    await DB.query(query, [blockHeight]);
   }
 }
 

backend/src/api/liquid/icons.ts

@@ -0,0 +1,38 @@
+import * as fs from 'fs';
+import logger from '../../logger';
+
+class Icons {
+  private static FILE_NAME = './icons.json';
+  private iconIds: string[] = [];
+  private icons: { [assetId: string]: string; } = {};
+
+  constructor() {}
+
+  public loadIcons() {
+    if (!fs.existsSync(Icons.FILE_NAME)) {
+      logger.warn(`${Icons.FILE_NAME} does not exist. No Liquid icons loaded.`);
+      return;
+    }
+    const cacheData = fs.readFileSync(Icons.FILE_NAME, 'utf8');
+    this.icons = JSON.parse(cacheData);
+
+    for (const i in this.icons) {
+      this.iconIds.push(i);
+    }
+    logger.debug(`Liquid icons has been loaded.`);
+  }
+
+  public getIconByAssetId(assetId: string): Buffer | undefined {
+    const icon = this.icons[assetId];
+    if (icon) {
+      return Buffer.from(icon, 'base64');
+    }
+  }
+
+  public getAllIconIds() {
+    return this.iconIds;
+  }
+
+}
+
+export default new Icons();

backend/src/api/loading-indicators.ts

@@ -12,8 +12,8 @@ class LoadingIndicators {
     this.progressChangedCallback = fn;
   }
 
-  public setProgress(name: string, progressPercent: number) {
-    const newProgress = Math.round(progressPercent);
+  public setProgress(name: string, progressPercent: number, rounded: boolean = true) {
+    const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent;
     if (newProgress >= 100) {
       delete this.loadingIndicators[name];
     } else {

backend/src/api/mempool-blocks.ts

@@ -1,10 +1,11 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
+  private mempoolBlockDeltas: MempoolBlockDelta[] = [];
 
   constructor() {}
 
@@ -25,6 +26,10 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
+  public getMempoolBlockDeltas(): MempoolBlockDelta[] {
+    return this.mempoolBlockDeltas;
+  }
+
   public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
     const latestMempool = memPool;
     const memPoolArray: TransactionExtended[] = [];
@@ -66,11 +71,15 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    this.mempoolBlocks = this.calculateMempoolBlocks(memPoolArray);
+    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    this.mempoolBlocks = blocks;
+    this.mempoolBlockDeltas = deltas;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
+    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     let blockWeight = 0;
     let blockSize = 0;
     let transactions: TransactionExtended[] = [];
@@ -90,7 +99,43 @@ class MempoolBlocks {
     if (transactions.length) {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
-    return mempoolBlocks;
+    // Calculate change from previous block states
+    for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
+      let added: TransactionStripped[] = [];
+      let removed: string[] = [];
+      if (mempoolBlocks[i] && !prevBlocks[i]) {
+        added = mempoolBlocks[i].transactions;
+      } else if (!mempoolBlocks[i] && prevBlocks[i]) {
+        removed = prevBlocks[i].transactions.map(tx => tx.txid);
+      } else if (mempoolBlocks[i] && prevBlocks[i]) {
+        const prevIds = {};
+        const newIds = {};
+        prevBlocks[i].transactions.forEach(tx => {
+          prevIds[tx.txid] = true;
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          newIds[tx.txid] = true;
+        });
+        prevBlocks[i].transactions.forEach(tx => {
+          if (!newIds[tx.txid]) {
+            removed.push(tx.txid);
+          }
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          if (!prevIds[tx.txid]) {
+            added.push(tx);
+          }
+        });
+      }
+      mempoolBlockDeltas.push({
+        added,
+        removed
+      });
+    }
+    return {
+      blocks: mempoolBlocks,
+      deltas: mempoolBlockDeltas
+    };
   }
 
   private dataToMempoolBlocks(transactions: TransactionExtended[],
@@ -112,6 +157,7 @@ class MempoolBlocks {
       medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
+      transactions: transactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
 }

backend/src/api/mempool.ts

@@ -8,13 +8,15 @@ import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
 import loadingIndicators from './loading-indicators';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import rbfCache from './rbf-cache';
 
 class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
   private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
+  private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: TransactionExtended } = {};
-  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
+  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
@@ -32,6 +34,17 @@ class Mempool {
     setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
+  /**
+   * Return true if we should leave resources available for mempool tx caching
+   */
+  public hasPriority(): boolean {
+    if (this.inSync) {
+      return false;
+    } else {
+      return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
+    }
+  }
+
   public isInSync(): boolean {
     return this.inSync;
   }
@@ -100,6 +113,8 @@ class Mempool {
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
+    this.mempoolCacheDelta = Math.abs(diff);
+
     if (!this.inSync) {
       loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
     }
@@ -174,6 +189,8 @@ class Mempool {
       loadingIndicators.setProgress('mempool', 100);
     }
 
+    this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
+
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
@@ -184,6 +201,17 @@ class Mempool {
     logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
   }
 
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
+    for (const rbfTransaction in rbfTransactions) {
+      if (this.mempoolCache[rbfTransaction]) {
+        // Store replaced transactions
+        rbfCache.add(rbfTransaction, rbfTransactions[rbfTransaction].txid);
+        // Erase the replaced transactions from the local mempool
+        delete this.mempoolCache[rbfTransaction];
+      }
+    }
+  }
+
   private updateTxPerSecond() {
     const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
     this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);

backend/src/api/mining.ts

@@ -0,0 +1,481 @@
+import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
+import BlocksRepository from '../repositories/BlocksRepository';
+import PoolsRepository from '../repositories/PoolsRepository';
+import HashratesRepository from '../repositories/HashratesRepository';
+import bitcoinClient from './bitcoin/bitcoin-client';
+import logger from '../logger';
+import { Common } from './common';
+import loadingIndicators from './loading-indicators';
+import { escape } from 'mysql2';
+import indexer from '../indexer';
+import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import config from '../config';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+
+class Mining {
+  constructor() {
+  }
+
+  /**
+   * Get historical block predictions match rate
+   */
+   public async $getBlockPredictionsHistory(interval: string | null = null): Promise<any> {
+    return await BlocksAuditsRepository.$getBlockPredictionsHistory(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block total fee
+   */
+  public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockFees(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block rewards
+   */
+  public async $getHistoricalBlockRewards(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockRewards(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block fee rates percentiles
+   */
+   public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockFeeRates(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block sizes
+   */
+   public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockSizes(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block weights
+   */
+   public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockWeights(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Generate high level overview of the pool ranks and general stats
+   */
+  public async $getPoolsStats(interval: string | null): Promise<object> {
+    const poolsStatistics = {};
+
+    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
+    const emptyBlocks: any[] = await BlocksRepository.$countEmptyBlocks(null, interval);
+
+    const poolsStats: PoolStats[] = [];
+    let rank = 1;
+
+    poolsInfo.forEach((poolInfo: PoolInfo) => {
+      const emptyBlocksCount = emptyBlocks.filter((emptyCount) => emptyCount.poolId === poolInfo.poolId);
+      const poolStat: PoolStats = {
+        poolId: poolInfo.poolId, // mysql row id
+        name: poolInfo.name,
+        link: poolInfo.link,
+        blockCount: poolInfo.blockCount,
+        rank: rank++,
+        emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
+        slug: poolInfo.slug,
+      };
+      poolsStats.push(poolStat);
+    });
+
+    poolsStatistics['pools'] = poolsStats;
+
+    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
+    poolsStatistics['blockCount'] = blockCount;
+
+    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+
+    try {
+      poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      poolsStatistics['lastEstimatedHashrate'] = 0;
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }
+
+    return poolsStatistics;
+  }
+
+  /**
+   * Get all mining pool stats for a pool
+   */
+  public async $getPoolStat(slug: string): Promise<object> {
+    const pool = await PoolsRepository.$getPool(slug);
+    if (!pool) {
+      throw new Error('This mining pool does not exist ' + escape(slug));
+    }
+
+    const blockCount: number = await BlocksRepository.$blockCount(pool.id);
+    const totalBlock: number = await BlocksRepository.$blockCount(null, null);
+
+    const blockCount24h: number = await BlocksRepository.$blockCount(pool.id, '24h');
+    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+
+    const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
+    const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
+
+    let currentEstimatedHashrate = 0;
+    try {
+      currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }
+
+    return {
+      pool: pool,
+      blockCount: {
+        'all': blockCount,
+        '24h': blockCount24h,
+        '1w': blockCount1w,
+      },
+      blockShare: {
+        'all': blockCount / totalBlock,
+        '24h': blockCount24h / totalBlock24h,
+        '1w': blockCount1w / totalBlock1w,
+      },
+      estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
+      reportedHashrate: null,
+    };
+  }
+
+  /**
+   * Get miner reward stats
+   */
+  public async $getRewardStats(blockCount: number): Promise<RewardStats> {
+    return await BlocksRepository.$getBlockStats(blockCount);
+  }
+
+  /**
+   * [INDEXING] Generate weekly mining pool hashrate history
+   */
+  public async $generatePoolHashrateHistory(): Promise<void> {
+    const now = new Date();
+    const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
+
+    // Run only if:
+    // * lastestRunDate is set to 0 (node backend restart, reorg)
+    // * we started a new week (around Monday midnight)
+    const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
+    if (!runIndexing) {
+      return;
+    }
+
+    try {
+      const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
+
+      const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
+      const hashrates: any[] = [];
+ 
+      const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
+      const lastMondayMidnight = this.getDateMidnight(lastMonday);
+      let toTimestamp = lastMondayMidnight.getTime();
+
+      const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008;
+      let indexedThisRun = 0;
+      let totalIndexed = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      logger.debug(`Indexing weekly mining pool hashrate`);
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
+
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
+        const fromTimestamp = toTimestamp - 604800000;
+
+        // Skip already indexed weeks
+        if (indexedTimestamp.includes(toTimestamp / 1000)) {
+          toTimestamp -= 604800000;
+          ++totalIndexed;
+          continue;
+        }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp / 1000, toTimestamp / 1000);
+        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
+        const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
+        if (totalBlocks > 0) {
+          pools = pools.map((pool: any) => {
+            pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
+            pool.share = (pool.blockCount / totalBlocks);
+            return pool;
+          });
+
+          for (const pool of pools) {
+            hashrates.push({
+              hashrateTimestamp: toTimestamp / 1000,
+              avgHashrate: pool['hashrate'] ,
+              poolId: pool.poolId,
+              share: pool['share'],
+              type: 'weekly',
+            });
+          }
+
+          newlyIndexed += hashrates.length;
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
+
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 1) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
+          const formattedDate = new Date(fromTimestamp).toUTCString();
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+          loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
+        }
+
+        toTimestamp -= 604800000;
+        ++indexedThisRun;
+        ++totalIndexed;
+      }
+      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
+      if (newlyIndexed > 0) {
+        logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
+      } else {
+        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
+      }
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+    } catch (e) {
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Generate daily hashrate data
+   */
+  public async $generateNetworkHashrateHistory(): Promise<void> {
+    // We only run this once a day around midnight
+    const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
+    const now = new Date().getUTCDate();
+    if (now === latestRunDate) {
+      return;
+    }
+
+    const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
+    try {
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
+      const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
+      const lastMidnight = this.getDateMidnight(new Date());
+      let toTimestamp = Math.round(lastMidnight.getTime());
+      const hashrates: any[] = [];
+
+      const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
+      let indexedThisRun = 0;
+      let totalIndexed = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      logger.debug(`Indexing daily network hashrate`);
+      loadingIndicators.setProgress('daily-hashrate-indexing', 0);
+
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
+        const fromTimestamp = toTimestamp - 86400000;
+
+        // Skip already indexed days
+        if (indexedTimestamp.includes(toTimestamp / 1000)) {
+          toTimestamp -= 86400000;
+          ++totalIndexed;
+          continue;
+        }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp / 1000, toTimestamp / 1000);
+        const lastBlockHashrate = blockStats.blockCount === 0 ? 0 : await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        hashrates.push({
+          hashrateTimestamp: toTimestamp / 1000,
+          avgHashrate: lastBlockHashrate,
+          poolId: 0,
+          share: 1,
+          type: 'daily',
+        });
+
+        if (hashrates.length > 10) {
+          newlyIndexed += hashrates.length;
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
+
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 1) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
+          const formattedDate = new Date(fromTimestamp).toUTCString();
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+          loadingIndicators.setProgress('daily-hashrate-indexing', progress);
+        }
+
+        toTimestamp -= 86400000;
+        ++indexedThisRun;
+        ++totalIndexed;
+      }
+
+      // Add genesis block manually
+      if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && !indexedTimestamp.includes(genesisTimestamp / 1000)) {
+        hashrates.push({
+          hashrateTimestamp: genesisTimestamp / 1000,
+          avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
+          poolId: 0,
+          share: 1,
+          type: 'daily',
+        });
+      }
+
+      newlyIndexed += hashrates.length;
+      await HashratesRepository.$saveHashrates(hashrates);
+
+      await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
+      if (newlyIndexed > 0) {
+        logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
+      } else {
+        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
+      }
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+    } catch (e) {
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  /**
+   * Index difficulty adjustments
+   */
+  public async $indexDifficultyAdjustments(): Promise<void> {
+    const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
+    const indexedHeights = {};
+    for (const height of indexedHeightsArray) {
+      indexedHeights[height] = true;
+    }
+
+    const blocks: any = await BlocksRepository.$getBlocksDifficulty();
+    const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+    let currentDifficulty = genesisBlock.difficulty;
+    let totalIndexed = 0;
+
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
+      await DifficultyAdjustmentsRepository.$saveAdjustments({
+        time: genesisBlock.time,
+        height: 0,
+        difficulty: currentDifficulty,
+        adjustment: 0.0,
+      });
+    }
+
+    const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock();
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) {
+      currentDifficulty = oldestConsecutiveBlock.difficulty;
+    }
+
+    let totalBlockChecked = 0;
+    let timer = new Date().getTime() / 1000;
+
+    for (const block of blocks) {
+      if (block.difficulty !== currentDifficulty) {
+        if (indexedHeights[block.height] === true) { // Already indexed
+          if (block.height >= oldestConsecutiveBlock.height) {
+            currentDifficulty = block.difficulty;
+          }
+          continue;          
+        }
+
+        let adjustment = block.difficulty / currentDifficulty;
+        adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
+
+        await DifficultyAdjustmentsRepository.$saveAdjustments({
+          time: block.time,
+          height: block.height,
+          difficulty: block.difficulty,
+          adjustment: adjustment,
+        });
+
+        totalIndexed++;
+        if (block.height >= oldestConsecutiveBlock.height) {
+          currentDifficulty = block.difficulty;
+        }
+    }
+
+      totalBlockChecked++;
+      const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+      if (elapsedSeconds > 5) {
+        const progress = Math.round(totalBlockChecked / blocks.length * 100);
+        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`);
+        timer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (totalIndexed > 0) {
+      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
+    } else {
+      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`);
+    }
+  }
+
+  private getDateMidnight(date: Date): Date {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+
+    return date;
+  }
+
+  private getTimeRange(interval: string | null): number {
+    switch (interval) {
+      case '3y': return 43200; // 12h
+      case '2y': return 28800; // 8h
+      case '1y': return 28800; // 8h
+      case '6m': return 10800; // 3h
+      case '3m': return 7200; // 2h
+      case '1m': return 1800; // 30min
+      case '1w': return 300; // 5min
+      case '3d': return 1;
+      case '24h': return 1;
+      default: return 86400; // 24h
+    }
+  }
+}
+
+export default new Mining();

backend/src/api/pools-parser.ts

@@ -0,0 +1,252 @@
+import DB from '../database';
+import logger from '../logger';
+import config from '../config';
+import BlocksRepository from '../repositories/BlocksRepository';
+
+interface Pool {
+  name: string;
+  link: string;
+  regexes: string[];
+  addresses: string[];
+  slug: string;
+}
+
+class PoolsParser {
+  miningPools: any[] = [];
+  unknownPool: any = {
+    'name': "Unknown",
+    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
+    'regexes': "[]",
+    'addresses': "[]",
+    'slug': 'unknown'
+  };
+  slugWarnFlag = false;
+
+  /**
+   * Parse the pools.json file, consolidate the data and dump it into the database
+   */
+  public async migratePoolsJson(poolsJson: object) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return;
+    }
+
+    // First we save every entries without paying attention to pool duplication
+    const poolsDuplicated: Pool[] = [];
+
+    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
+    for (let i = 0; i < coinbaseTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>coinbaseTags[i][1]).name,
+        'link': (<Pool>coinbaseTags[i][1]).link,
+        'regexes': [coinbaseTags[i][0]],
+        'addresses': [],
+        'slug': ''
+      });
+    }
+    const addressesTags = Object.entries(poolsJson['payout_addresses']);
+    for (let i = 0; i < addressesTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>addressesTags[i][1]).name,
+        'link': (<Pool>addressesTags[i][1]).link,
+        'regexes': [],
+        'addresses': [addressesTags[i][0]],
+        'slug': ''
+      });
+    }
+
+    // Then, we find unique mining pool names
+    const poolNames: string[] = [];
+    for (let i = 0; i < poolsDuplicated.length; ++i) {
+      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
+        poolNames.push(poolsDuplicated[i].name);
+      }
+    }
+    logger.debug(`Found ${poolNames.length} unique mining pools`);
+
+    // Get existing pools from the db
+    let existingPools;
+    try {
+      if (config.DATABASE.ENABLED === true) {
+        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      } else {
+        existingPools = [];
+      }
+    } catch (e) {
+      logger.err('Cannot get existing pools from the database, skipping pools.json import');
+      return;
+    }
+
+    this.miningPools = [];
+
+    // Finally, we generate the final consolidated pools data
+    const finalPoolDataAdd: Pool[] = [];
+    const finalPoolDataUpdate: Pool[] = [];
+    for (let i = 0; i < poolNames.length; ++i) {
+      let allAddresses: string[] = [];
+      let allRegexes: string[] = [];
+      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
+
+      for (let y = 0; y < match.length; ++y) {
+        allAddresses = allAddresses.concat(match[y].addresses);
+        allRegexes = allRegexes.concat(match[y].regexes);
+      }
+
+      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
+
+      let slug: string | undefined;
+      try {
+        slug = poolsJson['slugs'][poolNames[i]];
+      } catch (e) {
+        if (this.slugWarnFlag === false) {
+          logger.warn(`pools.json does not seem to contain the 'slugs' object`);
+          this.slugWarnFlag = true;
+        }
+      }
+
+      if (slug === undefined) {
+        // Only keep alphanumerical
+        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
+      }
+
+      const poolObj = {
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': allRegexes,
+        'addresses': allAddresses,
+        'slug': slug
+      };
+
+      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
+      if (existingPool !== undefined) {
+        // Check if any data was actually updated
+        const equals = (a, b) =>
+          a.length === b.length &&
+          a.every((v, i) => v === b[i]);
+        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
+          finalPoolDataUpdate.push(poolObj);
+        }
+      } else {
+        logger.debug(`Add '${finalPoolName}' mining pool`);
+        finalPoolDataAdd.push(poolObj);
+      }
+
+      this.miningPools.push({
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': JSON.stringify(allRegexes),
+        'addresses': JSON.stringify(allAddresses),
+        'slug': slug
+      });
+    }
+
+    if (config.DATABASE.ENABLED === false) { // Don't run db operations
+      logger.info('Mining pools.json import completed (no database)');
+      return;
+    }
+
+    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0) {    
+      logger.debug(`Update pools table now`);
+
+      // Add new mining pools into the database
+      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
+      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
+        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
+        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
+        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
+      }
+      queryAdd = queryAdd.slice(0, -1) + ';';
+
+      // Updated existing mining pools in the database
+      const updateQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
+        updateQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
+          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
+          slug='${finalPoolDataUpdate[i].slug}'
+          WHERE name='${finalPoolDataUpdate[i].name}'
+        ;`);
+      }
+
+      try {
+        await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+
+        if (finalPoolDataAdd.length > 0) {
+          await DB.query({ sql: queryAdd, timeout: 120000 });
+        }
+        for (const query of updateQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
+        await this.insertUnknownPool();
+        logger.info('Mining pools.json import completed');
+      } catch (e) {
+        logger.err(`Cannot import pools in the database`);
+        throw e;
+      }
+    }
+
+    try {
+      await this.insertUnknownPool();
+    } catch (e) {
+      logger.err(`Cannot insert unknown pool in the database`);
+      throw e;
+    }
+  }
+
+  /**
+   * Manually add the 'unknown pool'
+   */
+  private async insertUnknownPool() {
+    try {
+      const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
+      if (rows.length === 0) {
+        await DB.query({
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
+          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+        `});
+      } else {
+        await DB.query(`UPDATE pools
+          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
+          regexes='[]', addresses='[]',
+          slug='unknown'
+          WHERE name='Unknown'
+        `);
+      }
+    } catch (e) {
+      logger.err('Unable to insert "Unknown" mining pool');
+    }
+  }
+
+  /**
+   * Delete blocks which needs to be reindexed
+   */
+   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+    if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
+      return;
+    }
+
+    const blockCount = await BlocksRepository.$blockCount(null, null);
+    if (blockCount === 0) {
+      return;
+    }
+
+    for (const updatedPool of finalPoolDataUpdate) {
+      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
+      if (pool.length > 0) {
+        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`);
+        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
+      }
+    }
+
+    // Ignore early days of Bitcoin as there were not mining pool yet
+    logger.notice('Deleting blocks with unknown mining pool from height 130635 for future re-indexing');
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+
+    logger.notice('Truncating hashrates for future re-indexing');
+    await DB.query(`DELETE FROM hashrates`);
+  }
+}
+
+export default new PoolsParser();

backend/src/api/rbf-cache.ts

@@ -0,0 +1,34 @@
+export interface CachedRbf {
+  txid: string;
+  expires: Date;
+}
+
+class RbfCache {
+  private cache: { [txid: string]: CachedRbf; } = {};
+
+  constructor() {
+    setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
+  }
+
+  public add(replacedTxId: string, newTxId: string): void {
+    this.cache[replacedTxId] = {
+      expires: new Date(Date.now() + 1000 * 604800), // 1 week
+      txid: newTxId,
+    };
+  }
+
+  public get(txId: string): CachedRbf | undefined {
+    return this.cache[txId];
+  }
+
+  private cleanup(): void {
+    const currentDate = new Date();
+    for (const c in this.cache) {
+      if (this.cache[c].expires < currentDate) {
+        delete this.cache[c];
+      }
+    }
+  }
+}
+
+export default new RbfCache();

backend/src/api/statistics.ts

@@ -1,17 +1,15 @@
 import memPool from './mempool';
-import { DB } from '../database';
+import DB from '../database';
 import logger from '../logger';
 
 import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces';
 import config from '../config';
+import { Common } from './common';
 
 class Statistics {
   protected intervalTimer: NodeJS.Timer | undefined;
   protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined;
   protected queryTimeout = 120000;
-  protected cache: { [date: string]: OptimizedStatistic[] } = {
-    '24h': [], '1w': [], '1m': [], '3m': [], '6m': [], '1y': [], '2y': [], '3y': []
-  };
 
   public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) {
     this.newStatisticsEntryCallback = fn;
@@ -33,25 +31,6 @@ class Statistics {
         this.runStatistics();
       }, 1 * 60 * 1000);
     }, difference);
-
-    this.createCache();
-    setInterval(this.createCache.bind(this), 600000);
-  }
-
-  public getCache() {
-    return this.cache;
-  }
-
-  private async createCache() {
-    this.cache['24h'] = await this.$list24H();
-    this.cache['1w'] = await this.$list1W();
-    this.cache['1m'] = await this.$list1M();
-    this.cache['3m'] = await this.$list3M();
-    this.cache['6m'] = await this.$list6M();
-    this.cache['1y'] = await this.$list1Y();
-    this.cache['2y'] = await this.$list2Y();
-    this.cache['3y'] = await this.$list3Y();
-    logger.debug('Statistics cache created');
   }
 
   private async runStatistics(): Promise<void> {
@@ -74,6 +53,17 @@ class Statistics {
     memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
 
     if (!memPoolArray.length) {
+      try {
+        const insertIdZeroed = await this.$createZeroedStatistic();
+        if (this.newStatisticsEntryCallback && insertIdZeroed) {
+          const newStats = await this.$get(insertIdZeroed);
+          if (newStats) {
+            this.newStatisticsEntryCallback(newStats);
+          }
+        }
+      } catch (e) {
+        logger.err('Unable to insert zeroed statistics. ' + e);
+      }
       return;
     }
 
@@ -90,9 +80,9 @@ class Statistics {
     memPoolArray.forEach((transaction) => {
       for (let i = 0; i < logFees.length; i++) {
         if (
-          (config.MEMPOOL.NETWORK === 'liquid' && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1]))
+          (Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1]))
           ||
-          (config.MEMPOOL.NETWORK !== 'liquid' && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1]))
+          (!Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1]))
         ) {
           if (weightVsizeFees[logFees[i]]) {
             weightVsizeFees[logFees[i]] += transaction.vsize;
@@ -104,65 +94,126 @@ class Statistics {
       }
     });
 
-    const insertId = await this.$create({
-      added: 'NOW()',
-      unconfirmed_transactions: memPoolArray.length,
-      tx_per_second: txPerSecond,
-      vbytes_per_second: Math.round(vBytesPerSecond),
-      mempool_byte_weight: totalWeight,
-      total_fee: totalFee,
-      fee_data: '',
-      vsize_1: weightVsizeFees['1'] || 0,
-      vsize_2: weightVsizeFees['2'] || 0,
-      vsize_3: weightVsizeFees['3'] || 0,
-      vsize_4: weightVsizeFees['4'] || 0,
-      vsize_5: weightVsizeFees['5'] || 0,
-      vsize_6: weightVsizeFees['6'] || 0,
-      vsize_8: weightVsizeFees['8'] || 0,
-      vsize_10: weightVsizeFees['10'] || 0,
-      vsize_12: weightVsizeFees['12'] || 0,
-      vsize_15: weightVsizeFees['15'] || 0,
-      vsize_20: weightVsizeFees['20'] || 0,
-      vsize_30: weightVsizeFees['30'] || 0,
-      vsize_40: weightVsizeFees['40'] || 0,
-      vsize_50: weightVsizeFees['50'] || 0,
-      vsize_60: weightVsizeFees['60'] || 0,
-      vsize_70: weightVsizeFees['70'] || 0,
-      vsize_80: weightVsizeFees['80'] || 0,
-      vsize_90: weightVsizeFees['90'] || 0,
-      vsize_100: weightVsizeFees['100'] || 0,
-      vsize_125: weightVsizeFees['125'] || 0,
-      vsize_150: weightVsizeFees['150'] || 0,
-      vsize_175: weightVsizeFees['175'] || 0,
-      vsize_200: weightVsizeFees['200'] || 0,
-      vsize_250: weightVsizeFees['250'] || 0,
-      vsize_300: weightVsizeFees['300'] || 0,
-      vsize_350: weightVsizeFees['350'] || 0,
-      vsize_400: weightVsizeFees['400'] || 0,
-      vsize_500: weightVsizeFees['500'] || 0,
-      vsize_600: weightVsizeFees['600'] || 0,
-      vsize_700: weightVsizeFees['700'] || 0,
-      vsize_800: weightVsizeFees['800'] || 0,
-      vsize_900: weightVsizeFees['900'] || 0,
-      vsize_1000: weightVsizeFees['1000'] || 0,
-      vsize_1200: weightVsizeFees['1200'] || 0,
-      vsize_1400: weightVsizeFees['1400'] || 0,
-      vsize_1600: weightVsizeFees['1600'] || 0,
-      vsize_1800: weightVsizeFees['1800'] || 0,
-      vsize_2000: weightVsizeFees['2000'] || 0,
-    });
+    try {
+      const insertId = await this.$create({
+        added: 'NOW()',
+        unconfirmed_transactions: memPoolArray.length,
+        tx_per_second: txPerSecond,
+        vbytes_per_second: Math.round(vBytesPerSecond),
+        mempool_byte_weight: totalWeight,
+        total_fee: totalFee,
+        fee_data: '',
+        vsize_1: weightVsizeFees['1'] || 0,
+        vsize_2: weightVsizeFees['2'] || 0,
+        vsize_3: weightVsizeFees['3'] || 0,
+        vsize_4: weightVsizeFees['4'] || 0,
+        vsize_5: weightVsizeFees['5'] || 0,
+        vsize_6: weightVsizeFees['6'] || 0,
+        vsize_8: weightVsizeFees['8'] || 0,
+        vsize_10: weightVsizeFees['10'] || 0,
+        vsize_12: weightVsizeFees['12'] || 0,
+        vsize_15: weightVsizeFees['15'] || 0,
+        vsize_20: weightVsizeFees['20'] || 0,
+        vsize_30: weightVsizeFees['30'] || 0,
+        vsize_40: weightVsizeFees['40'] || 0,
+        vsize_50: weightVsizeFees['50'] || 0,
+        vsize_60: weightVsizeFees['60'] || 0,
+        vsize_70: weightVsizeFees['70'] || 0,
+        vsize_80: weightVsizeFees['80'] || 0,
+        vsize_90: weightVsizeFees['90'] || 0,
+        vsize_100: weightVsizeFees['100'] || 0,
+        vsize_125: weightVsizeFees['125'] || 0,
+        vsize_150: weightVsizeFees['150'] || 0,
+        vsize_175: weightVsizeFees['175'] || 0,
+        vsize_200: weightVsizeFees['200'] || 0,
+        vsize_250: weightVsizeFees['250'] || 0,
+        vsize_300: weightVsizeFees['300'] || 0,
+        vsize_350: weightVsizeFees['350'] || 0,
+        vsize_400: weightVsizeFees['400'] || 0,
+        vsize_500: weightVsizeFees['500'] || 0,
+        vsize_600: weightVsizeFees['600'] || 0,
+        vsize_700: weightVsizeFees['700'] || 0,
+        vsize_800: weightVsizeFees['800'] || 0,
+        vsize_900: weightVsizeFees['900'] || 0,
+        vsize_1000: weightVsizeFees['1000'] || 0,
+        vsize_1200: weightVsizeFees['1200'] || 0,
+        vsize_1400: weightVsizeFees['1400'] || 0,
+        vsize_1600: weightVsizeFees['1600'] || 0,
+        vsize_1800: weightVsizeFees['1800'] || 0,
+        vsize_2000: weightVsizeFees['2000'] || 0,
+      });
 
-    if (this.newStatisticsEntryCallback && insertId) {
-      const newStats = await this.$get(insertId);
-      if (newStats) {
-        this.newStatisticsEntryCallback(newStats);
+      if (this.newStatisticsEntryCallback && insertId) {
+        const newStats = await this.$get(insertId);
+        if (newStats) {
+          this.newStatisticsEntryCallback(newStats);
+        }
       }
+    } catch (e) {
+      logger.err('Unable to insert statistics. ' + e);
+    }
+  }
+
+  private async $createZeroedStatistic(): Promise<number | undefined> {
+    try {
+      const query = `INSERT INTO statistics(
+              added,
+              unconfirmed_transactions,
+              tx_per_second,
+              vbytes_per_second,
+              mempool_byte_weight,
+              fee_data,
+              total_fee,
+              vsize_1,
+              vsize_2,
+              vsize_3,
+              vsize_4,
+              vsize_5,
+              vsize_6,
+              vsize_8,
+              vsize_10,
+              vsize_12,
+              vsize_15,
+              vsize_20,
+              vsize_30,
+              vsize_40,
+              vsize_50,
+              vsize_60,
+              vsize_70,
+              vsize_80,
+              vsize_90,
+              vsize_100,
+              vsize_125,
+              vsize_150,
+              vsize_175,
+              vsize_200,
+              vsize_250,
+              vsize_300,
+              vsize_350,
+              vsize_400,
+              vsize_500,
+              vsize_600,
+              vsize_700,
+              vsize_800,
+              vsize_900,
+              vsize_1000,
+              vsize_1200,
+              vsize_1400,
+              vsize_1600,
+              vsize_1800,
+              vsize_2000
+            )
+            VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
+      const [result]: any = await DB.query(query);
+      return result.insertId;
+    } catch (e) {
+      logger.err('$create() error' + (e instanceof Error ? e.message : e));
     }
   }
 
   private async $create(statistics: Statistic): Promise<number | undefined> {
     try {
-      const connection = await DB.pool.getConnection();
       const query = `INSERT INTO statistics(
               added,
               unconfirmed_transactions,
@@ -259,8 +310,7 @@ class Statistics {
         statistics.vsize_1800,
         statistics.vsize_2000,
       ];
-      const [result]: any = await connection.query(query, params);
-      connection.release();
+      const [result]: any = await DB.query(query, params);
       return result.insertId;
     } catch (e) {
       logger.err('$create() error' + (e instanceof Error ? e.message : e));
@@ -268,58 +318,57 @@ class Statistics {
   }
 
   private getQueryForDaysAvg(div: number, interval: string) {
-    return `SELECT id, UNIX_TIMESTAMP(added) as added,
-      CAST(avg(unconfirmed_transactions) as FLOAT) as unconfirmed_transactions,
-      CAST(avg(tx_per_second) as FLOAT) as tx_per_second,
-      CAST(avg(vbytes_per_second) as FLOAT) as vbytes_per_second,
-      CAST(avg(vsize_1) as FLOAT) as vsize_1,
-      CAST(avg(vsize_2) as FLOAT) as vsize_2,
-      CAST(avg(vsize_3) as FLOAT) as vsize_3,
-      CAST(avg(vsize_4) as FLOAT) as vsize_4,
-      CAST(avg(vsize_5) as FLOAT) as vsize_5,
-      CAST(avg(vsize_6) as FLOAT) as vsize_6,
-      CAST(avg(vsize_8) as FLOAT) as vsize_8,
-      CAST(avg(vsize_10) as FLOAT) as vsize_10,
-      CAST(avg(vsize_12) as FLOAT) as vsize_12,
-      CAST(avg(vsize_15) as FLOAT) as vsize_15,
-      CAST(avg(vsize_20) as FLOAT) as vsize_20,
-      CAST(avg(vsize_30) as FLOAT) as vsize_30,
-      CAST(avg(vsize_40) as FLOAT) as vsize_40,
-      CAST(avg(vsize_50) as FLOAT) as vsize_50,
-      CAST(avg(vsize_60) as FLOAT) as vsize_60,
-      CAST(avg(vsize_70) as FLOAT) as vsize_70,
-      CAST(avg(vsize_80) as FLOAT) as vsize_80,
-      CAST(avg(vsize_90) as FLOAT) as vsize_90,
-      CAST(avg(vsize_100) as FLOAT) as vsize_100,
-      CAST(avg(vsize_125) as FLOAT) as vsize_125,
-      CAST(avg(vsize_150) as FLOAT) as vsize_150,
-      CAST(avg(vsize_175) as FLOAT) as vsize_175,
-      CAST(avg(vsize_200) as FLOAT) as vsize_200,
-      CAST(avg(vsize_250) as FLOAT) as vsize_250,
-      CAST(avg(vsize_300) as FLOAT) as vsize_300,
-      CAST(avg(vsize_350) as FLOAT) as vsize_350,
-      CAST(avg(vsize_400) as FLOAT) as vsize_400,
-      CAST(avg(vsize_500) as FLOAT) as vsize_500,
-      CAST(avg(vsize_600) as FLOAT) as vsize_600,
-      CAST(avg(vsize_700) as FLOAT) as vsize_700,
-      CAST(avg(vsize_800) as FLOAT) as vsize_800,
-      CAST(avg(vsize_900) as FLOAT) as vsize_900,
-      CAST(avg(vsize_1000) as FLOAT) as vsize_1000,
-      CAST(avg(vsize_1200) as FLOAT) as vsize_1200,
-      CAST(avg(vsize_1400) as FLOAT) as vsize_1400,
-      CAST(avg(vsize_1600) as FLOAT) as vsize_1600,
-      CAST(avg(vsize_1800) as FLOAT) as vsize_1800,
-      CAST(avg(vsize_2000) as FLOAT) as vsize_2000 \
+    return `SELECT
+      UNIX_TIMESTAMP(added) as added,
+      CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
+      CAST(avg(vsize_1) as DOUBLE) as vsize_1,
+      CAST(avg(vsize_2) as DOUBLE) as vsize_2,
+      CAST(avg(vsize_3) as DOUBLE) as vsize_3,
+      CAST(avg(vsize_4) as DOUBLE) as vsize_4,
+      CAST(avg(vsize_5) as DOUBLE) as vsize_5,
+      CAST(avg(vsize_6) as DOUBLE) as vsize_6,
+      CAST(avg(vsize_8) as DOUBLE) as vsize_8,
+      CAST(avg(vsize_10) as DOUBLE) as vsize_10,
+      CAST(avg(vsize_12) as DOUBLE) as vsize_12,
+      CAST(avg(vsize_15) as DOUBLE) as vsize_15,
+      CAST(avg(vsize_20) as DOUBLE) as vsize_20,
+      CAST(avg(vsize_30) as DOUBLE) as vsize_30,
+      CAST(avg(vsize_40) as DOUBLE) as vsize_40,
+      CAST(avg(vsize_50) as DOUBLE) as vsize_50,
+      CAST(avg(vsize_60) as DOUBLE) as vsize_60,
+      CAST(avg(vsize_70) as DOUBLE) as vsize_70,
+      CAST(avg(vsize_80) as DOUBLE) as vsize_80,
+      CAST(avg(vsize_90) as DOUBLE) as vsize_90,
+      CAST(avg(vsize_100) as DOUBLE) as vsize_100,
+      CAST(avg(vsize_125) as DOUBLE) as vsize_125,
+      CAST(avg(vsize_150) as DOUBLE) as vsize_150,
+      CAST(avg(vsize_175) as DOUBLE) as vsize_175,
+      CAST(avg(vsize_200) as DOUBLE) as vsize_200,
+      CAST(avg(vsize_250) as DOUBLE) as vsize_250,
+      CAST(avg(vsize_300) as DOUBLE) as vsize_300,
+      CAST(avg(vsize_350) as DOUBLE) as vsize_350,
+      CAST(avg(vsize_400) as DOUBLE) as vsize_400,
+      CAST(avg(vsize_500) as DOUBLE) as vsize_500,
+      CAST(avg(vsize_600) as DOUBLE) as vsize_600,
+      CAST(avg(vsize_700) as DOUBLE) as vsize_700,
+      CAST(avg(vsize_800) as DOUBLE) as vsize_800,
+      CAST(avg(vsize_900) as DOUBLE) as vsize_900,
+      CAST(avg(vsize_1000) as DOUBLE) as vsize_1000,
+      CAST(avg(vsize_1200) as DOUBLE) as vsize_1200,
+      CAST(avg(vsize_1400) as DOUBLE) as vsize_1400,
+      CAST(avg(vsize_1600) as DOUBLE) as vsize_1600,
+      CAST(avg(vsize_1800) as DOUBLE) as vsize_1800,
+      CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \
       FROM statistics \
       WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
       GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
-      ORDER BY id DESC;`;
+      ORDER BY statistics.added DESC;`;
   }
 
   private getQueryForDays(div: number, interval: string) {
-    return `SELECT id, UNIX_TIMESTAMP(added) as added, unconfirmed_transactions,
-      tx_per_second,
-      vbytes_per_second,
+    return `SELECT
+      UNIX_TIMESTAMP(added) as added,
+      CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
       vsize_1,
       vsize_2,
       vsize_3,
@@ -361,15 +410,13 @@ class Statistics {
       FROM statistics \
       WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
       GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
-      ORDER BY id DESC;`;
+      ORDER BY statistics.added DESC;`;
   }
 
-  public async $get(id: number): Promise<OptimizedStatistic | undefined> {
+  private async $get(id: number): Promise<OptimizedStatistic | undefined> {
     try {
-      const connection = await DB.pool.getConnection();
       const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
-      const [rows] = await connection.query<any>(query, [id]);
-      connection.release();
+      const [rows] = await DB.query(query, [id]);
       if (rows[0]) {
         return this.mapStatisticToOptimizedStatistic([rows[0]])[0];
       }
@@ -380,11 +427,9 @@ class Statistics {
 
   public async $list2H(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY id DESC LIMIT 120`;
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`;
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list2H() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -393,11 +438,9 @@ class Statistics {
 
   public async $list24H(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDaysAvg(120, '1 DAY'); // 2m interval
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`;
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list24h() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -406,11 +449,9 @@ class Statistics {
 
   public async $list1W(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDaysAvg(600, '1 WEEK'); // 10m interval
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list1W() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -419,11 +460,9 @@ class Statistics {
 
   public async $list1M(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDaysAvg(3600, '1 MONTH'); // 1h interval
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list1M() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -432,11 +471,9 @@ class Statistics {
 
   public async $list3M(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDaysAvg(14400, '3 MONTH'); // 4h interval
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list3M() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -445,11 +482,9 @@ class Statistics {
 
   public async $list6M(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDaysAvg(21600, '6 MONTH'); // 6h interval 
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list6M() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -458,11 +493,9 @@ class Statistics {
 
   public async $list1Y(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(43200, '1 YEAR'); // 12h interval
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list1Y() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -471,11 +504,9 @@ class Statistics {
 
   public async $list2Y(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(86400, "2 YEAR"); // 1d interval
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = this.getQueryForDays(28800, '2 YEAR'); // 8h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list2Y() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -484,11 +515,9 @@ class Statistics {
 
   public async $list3Y(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(86400, "3 YEAR"); // 1d interval
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
+      const query = this.getQueryForDays(43200, '3 YEAR'); // 12h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
       logger.err('$list3Y() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -498,10 +527,7 @@ class Statistics {
   private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
     return statistic.map((s) => {
       return {
-        id: s.id || 0,
         added: s.added,
-        unconfirmed_transactions: s.unconfirmed_transactions,
-        tx_per_second: s.tx_per_second,
         vbytes_per_second: s.vbytes_per_second,
         mempool_byte_weight: s.mempool_byte_weight,
         total_fee: s.total_fee,

backend/src/api/transaction-utils.ts

@@ -2,6 +2,7 @@ import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import config from '../config';
+import { Common } from './common';
 
 class TransactionUtils {
   constructor() { }
@@ -20,8 +21,8 @@ class TransactionUtils {
     };
   }
 
-  public async $getTransactionExtended(txId: string, addPrevouts = false): Promise<TransactionExtended> {
-    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts);
+  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
+    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
     return this.extendTransaction(transaction);
   }
 
@@ -31,7 +32,8 @@ class TransactionUtils {
       // @ts-ignore
       return transaction;
     }
-    const feePerVbytes = Math.max(config.MEMPOOL.NETWORK === 'liquid' ? 0.1 : 1, (transaction.fee || 0) / (transaction.weight / 4));
+    const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
+      (transaction.fee || 0) / (transaction.weight / 4));
     const transactionExtended: TransactionExtended = Object.assign({
       vsize: Math.round(transaction.weight / 4),
       feePerVsize: feePerVbytes,
@@ -42,6 +44,14 @@ class TransactionUtils {
     }
     return transactionExtended;
   }
+
+  public hex2ascii(hex: string) {
+    let str = '';
+    for (let i = 0; i < hex.length; i += 2) {
+      str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
+    }
+    return str;
+  }
 }
 
 export default new TransactionUtils();

backend/src/api/websocket-handler.ts

@@ -1,7 +1,9 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
-import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock,
-  OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
+import {
+  BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+  OptimizedStatistic, ILoadingIndicators, IConversionRates
+} from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
@@ -11,6 +13,10 @@ import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import config from '../config';
 import transactionUtils from './transaction-utils';
+import rbfCache from './rbf-cache';
+import difficultyAdjustment from './difficulty-adjustment';
+import feeApi from './fee-api';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -48,29 +54,38 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-tx']) {
             if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
               client['track-tx'] = parsedMessage['track-tx'];
-              // Client is telling the transaction wasn't found but it might have appeared before we had the time to start watching for it
+              // Client is telling the transaction wasn't found
               if (parsedMessage['watch-mempool']) {
-                const tx = memPool.getMempool()[client['track-tx']];
-                if (tx) {
-                  if (config.MEMPOOL.BACKEND === 'esplora') {
-                    response['tx'] = tx;
+                const rbfCacheTx = rbfCache.get(client['track-tx']);
+                if (rbfCacheTx) {
+                  response['txReplaced'] = {
+                    txid: rbfCacheTx.txid,
+                  };
+                  client['track-tx'] = null;
+                } else {
+                  // It might have appeared before we had the time to start watching for it
+                  const tx = memPool.getMempool()[client['track-tx']];
+                  if (tx) {
+                    if (config.MEMPOOL.BACKEND === 'esplora') {
+                      response['tx'] = tx;
+                    } else {
+                      // tx.prevout is missing from transactions when in bitcoind mode
+                      try {
+                        const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
+                        response['tx'] = fullTx;
+                      } catch (e) {
+                        logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
+                      }
+                    }
                   } else {
-                    // tx.prevouts is missing from transactions when in bitcoind mode
                     try {
-                      const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
+                      const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
                       response['tx'] = fullTx;
                     } catch (e) {
-                      logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
+                      logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
+                      client['track-mempool-tx'] = parsedMessage['track-tx'];
                     }
                   }
-                } else {
-                  try {
-                    const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
-                    response['tx'] = fullTx;
-                  } catch (e) {
-                    logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
-                    client['track-mempool-tx'] = parsedMessage['track-tx'];
-                  }
                 }
               }
             } else {
@@ -99,6 +114,20 @@ class WebsocketHandler {
             }
           }
 
+          if (parsedMessage && parsedMessage['track-mempool-block'] !== undefined) {
+            if (Number.isInteger(parsedMessage['track-mempool-block']) && parsedMessage['track-mempool-block'] >= 0) {
+              const index = parsedMessage['track-mempool-block'];
+              client['track-mempool-block'] = index;
+              const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
+              response['projected-block-transactions'] = {
+                index: index,
+                blockTransactions: mBlocksWithTransactions[index]?.transactions || [],
+              };
+            } else {
+              client['track-mempool-block'] = null;
+            }
+          }
+
           if (parsedMessage.action === 'init') {
             const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
             if (!_blocks) {
@@ -138,7 +167,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -153,7 +182,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -166,7 +195,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -181,14 +210,14 @@ class WebsocketHandler {
     return {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
-      'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
-      'previousRetarget': blocks.getPreviousDifficultyRetarget(),
       'blocks': _blocks,
       'conversions': fiatConversion.getConversionRates(),
       'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
       'loadingIndicators': loadingIndicators.getLoadingIndicators(),
+      'da': difficultyAdjustment.getDifficultyAdjustment(),
+      'fees': feeApi.getRecommendedFee(),
       ...this.extraInitProperties
     };
   }
@@ -198,7 +227,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -221,16 +250,15 @@ class WebsocketHandler {
 
     mempoolBlocks.updateMempoolBlocks(newMempool);
     const mBlocks = mempoolBlocks.getMempoolBlocks();
-    const mempool = memPool.getMempool();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     const mempoolInfo = memPool.getMempoolInfo();
     const vBytesPerSecond = memPool.getVBytesPerSecond();
     const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
+    const da = difficultyAdjustment.getDifficultyAdjustment();
+    memPool.handleRbfTransactions(rbfTransactions);
+    const recommendedFees = feeApi.getRecommendedFee();
 
-    for (const rbfTransaction in rbfTransactions) {
-      delete mempool[rbfTransaction];
-    }
-
-    this.wss.clients.forEach(async (client: WebSocket) => {
+    this.wss.clients.forEach(async (client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -241,6 +269,8 @@ class WebsocketHandler {
         response['mempoolInfo'] = mempoolInfo;
         response['vBytesPerSecond'] = vBytesPerSecond;
         response['transactions'] = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
+        response['da'] = da;
+        response['fees'] = recommendedFees;
       }
 
       if (client['want-mempool-blocks']) {
@@ -331,22 +361,40 @@ class WebsocketHandler {
         }
       }
 
-      if (client['track-tx'] && rbfTransactions[client['track-tx']]) {
-        for (const rbfTransaction in rbfTransactions) {
-          if (client['track-tx'] === rbfTransaction) {
-            const rbfTx = rbfTransactions[rbfTransaction];
-            if (config.MEMPOOL.BACKEND !== 'esplora') {
-              try {
-                const fullTx = await transactionUtils.$getTransactionExtended(rbfTransaction, true);
-                response['rbfTransaction'] = fullTx;
-              } catch (e) {
-                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
-              }
-            } else {
-              response['rbfTransaction'] = rbfTx;
-            }
-            break;
+      if (client['track-tx']) {
+        const outspends: object = {};
+        newTransactions.forEach((tx) => tx.vin.forEach((vin, i) => {
+          if (vin.txid === client['track-tx']) {
+            outspends[vin.vout] = {
+              vin: i,
+              txid: tx.txid,
+            };
           }
+        }));
+
+        if (Object.keys(outspends).length) {
+          response['utxoSpent'] = outspends;
+        }
+
+        if (rbfTransactions[client['track-tx']]) {
+          for (const rbfTransaction in rbfTransactions) {
+            if (client['track-tx'] === rbfTransaction) {
+              response['rbfTransaction'] = {
+                txid: rbfTransactions[rbfTransaction].txid,
+              };
+              break;
+            }
+          }
+        }
+      }
+
+      if (client['track-mempool-block'] >= 0) {
+        const index = client['track-mempool-block'];
+        if (mBlockDeltas[index]) {
+          response['projected-block-transactions'] = {
+            index: index,
+            delta: mBlockDeltas[index],
+          };
         }
       }
 
@@ -362,25 +410,55 @@ class WebsocketHandler {
     }
 
     let mBlocks: undefined | MempoolBlock[];
+    let mBlockDeltas: undefined | MempoolBlockDelta[];
     let matchRate = 0;
     const _memPool = memPool.getMempool();
     const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
 
     if (_mempoolBlocks[0]) {
       const matches: string[] = [];
+      const added: string[] = [];
+      const missing: string[] = [];
+
       for (const txId of txIds) {
         if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
           matches.push(txId);
+        } else {
+          added.push(txId);
         }
         delete _memPool[txId];
       }
 
-      matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
+      for (const txId of _mempoolBlocks[0].transactionIds) {
+        if (matches.includes(txId) || added.includes(txId)) {
+          continue;
+        }
+        missing.push(txId);
+      }
+
+      matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
       mempoolBlocks.updateMempoolBlocks(_memPool);
       mBlocks = mempoolBlocks.getMempoolBlocks();
+      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
+      if (Common.indexingEnabled()) {
+        BlocksAuditsRepository.$saveAudit({
+          time: block.timestamp,
+          height: block.height,
+          hash: block.id,
+          addedTxs: added,
+          missingTxs: missing,
+          matchRate: matchRate,
+        });
+      }
     }
 
-    block.matchRate = matchRate;
+    if (block.extras) {
+      block.extras.matchRate = matchRate;
+    }
+
+    const da = difficultyAdjustment.getDifficultyAdjustment();
+    const fees = feeApi.getRecommendedFee();
 
     this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
@@ -394,8 +472,8 @@ class WebsocketHandler {
       const response = {
         'block': block,
         'mempoolInfo': memPool.getMempoolInfo(),
-        'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
-        'previousRetarget': blocks.getPreviousDifficultyRetarget(),
+        'da': da,
+        'fees': fees,
       };
 
       if (mBlocks && client['want-mempool-blocks']) {
@@ -403,7 +481,6 @@ class WebsocketHandler {
       }
 
       if (client['track-tx'] && txIds.indexOf(client['track-tx']) > -1) {
-        client['track-tx'] = null;
         response['txConfirmed'] = true;
       }
 
@@ -471,6 +548,16 @@ class WebsocketHandler {
         }
       }
 
+      if (client['track-mempool-block'] >= 0) {
+        const index = client['track-mempool-block'];
+        if (mBlockDeltas && mBlockDeltas[index]) {
+          response['projected-block-transactions'] = {
+            index: index,
+            delta: mBlockDeltas[index],
+          };
+        }
+      }
+
       client.send(JSON.stringify(response));
     });
   }

backend/src/config.ts

@@ -2,7 +2,7 @@ const configFile = require('../mempool-config.json');
 
 interface IConfig {
   MEMPOOL: {
-    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid';
+    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
     SPAWN_CLUSTER_PROCS: number;
@@ -14,8 +14,16 @@ interface IConfig {
     BLOCK_WEIGHT_UNITS: number;
     INITIAL_BLOCKS_AMOUNT: number;
     MEMPOOL_BLOCKS_AMOUNT: number;
+    INDEXING_BLOCKS_AMOUNT: number;
+    BLOCKS_SUMMARIES_INDEXING: boolean;
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
+    EXTERNAL_ASSETS: string[];
+    EXTERNAL_MAX_RETRY: number;
+    EXTERNAL_RETRY_INTERVAL: number;
+    USER_AGENT: string;
+    STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+    AUTOMATIC_BLOCK_REINDEXING: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -40,6 +48,7 @@ interface IConfig {
   DATABASE: {
     ENABLED: boolean;
     HOST: string,
+    SOCKET: string,
     PORT: number;
     DATABASE: string;
     USERNAME: string;
@@ -49,7 +58,7 @@ interface IConfig {
     ENABLED: boolean;
     HOST: string;
     PORT: number;
-    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' |'warn' | 'notice' | 'info' | 'debug';
+    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
     FACILITY: string;
   };
   STATISTICS: {
@@ -60,6 +69,26 @@ interface IConfig {
     ENABLED: boolean;
     DATA_PATH: string;
   };
+  SOCKS5PROXY: {
+    ENABLED: boolean;
+    USE_ONION: boolean;
+    HOST: string;
+    PORT: number;
+    USERNAME: string;
+    PASSWORD: string;
+  };
+  PRICE_DATA_SERVER: {
+    TOR_URL: string;
+    CLEARNET_URL: string;
+  };
+  EXTERNAL_DATA_SERVER: {
+    MEMPOOL_API: string;
+    MEMPOOL_ONION: string;
+    LIQUID_API: string;
+    LIQUID_ONION: string;
+    BISQ_URL: string;
+    BISQ_ONION: string;
+  };
 }
 
 const defaults: IConfig = {
@@ -76,8 +105,16 @@ const defaults: IConfig = {
     'BLOCK_WEIGHT_UNITS': 4000000,
     'INITIAL_BLOCKS_AMOUNT': 8,
     'MEMPOOL_BLOCKS_AMOUNT': 8,
-    'PRICE_FEED_UPDATE_INTERVAL': 3600,
+    'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
+    'BLOCKS_SUMMARIES_INDEXING': false,
+    'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
+    'EXTERNAL_ASSETS': [],
+    'EXTERNAL_MAX_RETRY': 1,
+    'EXTERNAL_RETRY_INTERVAL': 0,
+    'USER_AGENT': 'mempool',
+    'STDOUT_LOG_MIN_PRIORITY': 'debug',
+    'AUTOMATIC_BLOCK_REINDEXING': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -102,6 +139,7 @@ const defaults: IConfig = {
   'DATABASE': {
     'ENABLED': true,
     'HOST': '127.0.0.1',
+    'SOCKET': '',
     'PORT': 3306,
     'DATABASE': 'mempool',
     'USERNAME': 'mempool',
@@ -122,6 +160,26 @@ const defaults: IConfig = {
     'ENABLED': false,
     'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
   },
+  'SOCKS5PROXY': {
+    'ENABLED': false,
+    'USE_ONION': true,
+    'HOST': '127.0.0.1',
+    'PORT': 9050,
+    'USERNAME': '',
+    'PASSWORD': ''
+  },
+  "PRICE_DATA_SERVER": {
+    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
+  },
+  "EXTERNAL_DATA_SERVER": {
+    'MEMPOOL_API': 'https://mempool.space/api/v1',
+    'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+    'LIQUID_API': 'https://liquid.network/api/v1',
+    'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+    'BISQ_URL': 'https://bisq.markets/api',
+    'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+  }
 };
 
 class Config implements IConfig {
@@ -134,6 +192,9 @@ class Config implements IConfig {
   SYSLOG: IConfig['SYSLOG'];
   STATISTICS: IConfig['STATISTICS'];
   BISQ: IConfig['BISQ'];
+  SOCKS5PROXY: IConfig['SOCKS5PROXY'];
+  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
+  EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
 
   constructor() {
     const configs = this.merge(configFile, defaults);
@@ -146,6 +207,9 @@ class Config implements IConfig {
     this.SYSLOG = configs.SYSLOG;
     this.STATISTICS = configs.STATISTICS;
     this.BISQ = configs.BISQ;
+    this.SOCKS5PROXY = configs.SOCKS5PROXY;
+    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
+    this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/database.ts

@@ -1,26 +1,59 @@
 import config from './config';
-import { createPool } from 'mysql2/promise';
+import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
+import { PoolOptions } from 'mysql2/typings/mysql';
 
-export class DB {
-  static pool = createPool({
-    host: config.DATABASE.HOST,
+ class DB {
+  constructor() {
+    if (config.DATABASE.SOCKET !== '') {
+      this.poolConfig.socketPath = config.DATABASE.SOCKET;
+    } else {
+      this.poolConfig.host = config.DATABASE.HOST;
+    }
+  }
+  private pool: Pool | null = null;
+  private poolConfig: PoolOptions = {
     port: config.DATABASE.PORT,
     database: config.DATABASE.DATABASE,
     user: config.DATABASE.USERNAME,
     password: config.DATABASE.PASSWORD,
     connectionLimit: 10,
     supportBigNumbers: true,
-  });
-}
+    timezone: '+00:00',
+  };
 
-export async function checkDbConnection() {
-  try {
-    const connection = await DB.pool.getConnection();
-    logger.info('Database connection established.');
-    connection.release();
-  } catch (e) {
-    logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
-    process.exit(1);
+  private checkDBFlag() {
+    if (config.DATABASE.ENABLED === false) {
+      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+    }
+  }
+
+  public async query(query, params?) {
+    this.checkDBFlag();
+    const pool = await this.getPool();
+    return pool.query(query, params);
+  }
+
+  public async checkDbConnection() {
+    this.checkDBFlag();
+    try {
+      await this.query('SELECT ?', [1]);
+      logger.info('Database connection established.');
+    } catch (e) {
+      logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
+      process.exit(1);
+    }
+  }
+
+  private async getPool(): Promise<Pool> {
+    if (this.pool === null) {
+      this.pool = createPool(this.poolConfig);
+      this.pool.on('connection', function (newConnection: PoolConnection) {
+        newConnection.query(`SET time_zone='+00:00'`);
+      });
+    }
+    return this.pool;
   }
 }
+
+export default new DB();

backend/src/index.ts

@@ -1,11 +1,11 @@
-import { Express, Request, Response, NextFunction } from 'express';
-import * as express from 'express';
+import express from "express";
+import { Application, Request, Response, NextFunction, Express } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import axios from 'axios';
 
-import { checkDbConnection } from './database';
+import DB from './database';
 import config from './config';
 import routes from './routes';
 import blocks from './api/blocks';
@@ -21,11 +21,19 @@ import backendInfo from './api/backend-info';
 import loadingIndicators from './api/loading-indicators';
 import mempool from './api/mempool';
 import elementsParser from './api/liquid/elements-parser';
+import databaseMigration from './api/database-migration';
+import syncAssets from './sync-assets';
+import icons from './api/liquid/icons';
+import { Common } from './api/common';
+import poolsUpdater from './tasks/pools-updater';
+import indexer from './indexer';
+import priceUpdater from './tasks/price-updater';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
-  private app: Express;
+  private app: Application;
   private currentBackendRetryInterval = 5;
 
   constructor() {
@@ -36,7 +44,7 @@ class Server {
       return;
     }
 
-    if (cluster.isMaster) {
+    if (cluster.isPrimary) {
       logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
 
       const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
@@ -61,7 +69,7 @@ class Server {
   }
 
   async startServer(worker = false) {
-    logger.debug(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
+    logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
     this.app
       .use((req: Request, res: Response, next: NextFunction) => {
@@ -70,23 +78,46 @@ class Server {
       })
       .use(express.urlencoded({ extended: true }))
       .use(express.text())
-    ;
+      ;
 
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
 
     this.setUpWebsocketHandling();
 
+    await syncAssets.syncAssets$();
     diskCache.loadMempoolCache();
 
     if (config.DATABASE.ENABLED) {
-      await checkDbConnection();
+      await DB.checkDbConnection();
+      try {
+        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
+          const tables = process.env.npm_config_reindex.split(',');
+          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
+          await Common.sleep$(5000);
+          await databaseMigration.$truncateIndexedData(tables);
+        }
+        await databaseMigration.$initializeOrMigrateDatabase();
+        if (Common.indexingEnabled()) {
+          await indexer.$resetHashratesIndexingState();
+        }
+      } catch (e) {
+        throw new Error(e instanceof Error ? e.message : 'Error');
+      }
     }
 
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
       statistics.startStatistics();
     }
 
+    if (Common.isLiquid()) {
+      try {
+        icons.loadIcons();
+      } catch (e) {
+        logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
+      }
+    }
+
     fiatConversion.startService();
 
     this.setUpHttpApiRoutes();
@@ -120,8 +151,12 @@ class Server {
           logger.debug(msg);
         }
       }
+      await poolsUpdater.updatePoolsJson();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
+      indexer.$run();
+      priceUpdater.$run();
+
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
     } catch (e) {
@@ -143,7 +178,7 @@ class Server {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);
     }
-    if (config.MEMPOOL.NETWORK === 'liquid' && config.DATABASE.ENABLED) {
+    if (Common.isLiquid() && config.DATABASE.ENABLED) {
       blocks.setNewBlockCallback(async () => {
         try {
           await elementsParser.$parse();
@@ -163,6 +198,7 @@ class Server {
   setUpHttpApiRoutes() {
     this.app
       .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', routes.$getBatchedOutspends)
       .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', routes.getCpfpInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', routes.getDifficultyChange)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees)
@@ -173,7 +209,7 @@ class Server {
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -181,7 +217,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -191,7 +227,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -199,7 +235,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -207,19 +243,57 @@ class Server {
           res.status(500).end();
         }
       })
-    ;
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      ;
 
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
       this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', routes.get2HStatistics)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', routes.get24HStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', routes.get1WHStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', routes.get1MStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', routes.get3MStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', routes.get6MStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.get1YStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.get2YStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.get3YStatistics.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', routes.$getStatisticsByTime.bind(routes, '2h'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', routes.$getStatisticsByTime.bind(routes, '24h'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', routes.$getStatisticsByTime.bind(routes, '1w'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', routes.$getStatisticsByTime.bind(routes, '1m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', routes.$getStatisticsByTime.bind(routes, '3m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', routes.$getStatisticsByTime.bind(routes, '6m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y'))
+        ;
+    }
+
+    if (Common.indexingEnabled()) {
+      this.app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', routes.$getPools)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', routes.$getPoolHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', routes.$getPoolBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', routes.$getPoolBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', routes.$getDifficultyAdjustments)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', routes.$getHistoricalBlockPrediction)
         ;
     }
 
@@ -244,6 +318,12 @@ class Server {
         ;
     }
 
+    this.app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', routes.getStrippedBlockTransactions);
+
     if (config.MEMPOOL.BACKEND !== 'esplora') {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool)
@@ -254,11 +334,9 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', routes.getRawTransaction)
         .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus)
         .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks)
         .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', routes.getBlockTipHash)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', routes.getTxIdsForBlock)
@@ -267,13 +345,22 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix)
-      ;
+        ;
     }
 
-    if (config.MEMPOOL.NETWORK === 'liquid' && config.DATABASE.ENABLED) {
+    if (Common.isLiquid()) {
+      this.app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', routes.getAllLiquidIcon)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup)
+        ;
+    }
+
+    if (Common.isLiquid() && config.DATABASE.ENABLED) {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)
-      ;
+        ;
     }
   }
 }

backend/src/indexer.ts

@@ -0,0 +1,81 @@
+import { Common } from './api/common';
+import blocks from './api/blocks';
+import mempool from './api/mempool';
+import mining from './api/mining';
+import logger from './logger';
+import HashratesRepository from './repositories/HashratesRepository';
+import bitcoinClient from './api/bitcoin/bitcoin-client';
+
+class Indexer {
+  runIndexer = true;
+  indexerRunning = false;
+
+  constructor() {
+  }
+
+  public reindex() {
+    if (Common.indexingEnabled()) {
+      this.runIndexer = true;
+    }
+  }
+
+  public async $run() {
+    if (!Common.indexingEnabled() || this.runIndexer === false ||
+      this.indexerRunning === true || mempool.hasPriority()
+    ) {
+      return;
+    }
+
+    // Do not attempt to index anything unless Bitcoin Core is fully synced
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    if (blockchainInfo.blocks !== blockchainInfo.headers) {
+      return;
+    }
+
+    this.runIndexer = false;
+    this.indexerRunning = true;
+
+    logger.debug(`Running mining indexer`);
+
+    try {
+      const chainValid = await blocks.$generateBlockDatabase();
+      if (chainValid === false) {
+        // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
+        setTimeout(() => this.reindex(), 10000);
+        this.indexerRunning = false;
+        return;
+      }
+
+      await mining.$indexDifficultyAdjustments();
+      await this.$resetHashratesIndexingState();
+      await mining.$generateNetworkHashrateHistory();
+      await mining.$generatePoolHashrateHistory();
+      await blocks.$generateBlocksSummariesDatabase();
+    } catch (e) {
+      this.indexerRunning = false;
+      logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
+      setTimeout(() => this.reindex(), 10000);
+      this.indexerRunning = false;
+      return;
+    }
+
+    this.indexerRunning = false;
+
+    const runEvery = 1000 * 3600; // 1 hour
+    logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
+    setTimeout(() => this.reindex(), runEvery);
+  }
+
+  async $resetHashratesIndexingState() {
+    try {
+      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
+      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new Indexer();

backend/src/logger.ts

@@ -97,6 +97,9 @@ class Logger {
       syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
       this.syslog(syslogmsg);
     }
+    if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {
+      return;
+    }
     if (priority === 'warning') {
       priority = 'warn';
     }

backend/src/mempool.interfaces.ts

@@ -1,5 +1,36 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
 
+export interface PoolTag {
+  id: number; // mysql row id
+  name: string;
+  link: string;
+  regexes: string; // JSON array
+  addresses: string; // JSON array
+  slug: string;
+}
+
+export interface PoolInfo {
+  poolId: number; // mysql row id
+  name: string;
+  link: string;
+  blockCount: number;
+  slug: string;
+}
+
+export interface PoolStats extends PoolInfo {
+  rank: number;
+  emptyBlocks: number;
+}
+
+export interface BlockAudit {
+  time: number,
+  height: number,
+  hash: string,
+  missingTxs: string[],
+  addedTxs: string[],
+  matchRate: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;
@@ -11,6 +42,12 @@ export interface MempoolBlock {
 
 export interface MempoolBlockWithTransactions extends MempoolBlock {
   transactionIds: string[];
+  transactions: TransactionStripped[];
+}
+
+export interface MempoolBlockDelta {
+  added: TransactionStripped[];
+  removed: string[];
 }
 
 interface VinStrippedToScriptsig {
@@ -56,12 +93,31 @@ export interface TransactionStripped {
   vsize: number;
   value: number;
 }
-export interface BlockExtended extends IEsploraApi.Block {
+
+export interface BlockExtension {
+  totalFees?: number;
   medianFee?: number;
   feeRange?: number[];
   reward?: number;
   coinbaseTx?: TransactionMinerInfo;
   matchRate?: number;
+  pool?: {
+    id: number;
+    name: string;
+    slug: string;
+  };
+  avgFee?: number;
+  avgFeeRate?: number;
+  coinbaseRaw?: string;
+}
+
+export interface BlockExtended extends IEsploraApi.Block {
+  extras: BlockExtension;
+}
+
+export interface BlockSummary {
+  id: string;
+  transactions: TransactionStripped[];
 }
 
 export interface TransactionMinerInfo {
@@ -128,10 +184,7 @@ export interface Statistic {
 }
 
 export interface OptimizedStatistic {
-  id: number;
   added: string;
-  unconfirmed_transactions: number;
-  tx_per_second: number;
   vbytes_per_second: number;
   total_fee: number;
   mempool_byte_weight: number;
@@ -167,3 +220,28 @@ export interface IBackendInfo {
   gitCommit: string;
   version: string;
 }
+
+export interface IDifficultyAdjustment {
+  progressPercent: number;
+  difficultyChange: number;
+  estimatedRetargetDate: number;
+  remainingBlocks: number;
+  remainingTime: number;
+  previousRetarget: number;
+  nextRetargetHeight: number;
+  timeAvg: number;
+  timeOffset: number;
+}
+
+export interface IndexedDifficultyAdjustment {
+  time: number; // UNIX timestamp
+  height: number; // Block height
+  difficulty: number;
+  adjustment: number;
+}
+
+export interface RewardStats {
+  totalReward: number;
+  totalFee: number;
+  totalTx: number;
+}

backend/src/repositories/BlocksAuditsRepository.ts

@@ -0,0 +1,51 @@
+import DB from '../database';
+import logger from '../logger';
+import { BlockAudit } from '../mempool.interfaces';
+
+class BlocksAuditRepositories {
+  public async $saveAudit(audit: BlockAudit): Promise<void> {
+    try {
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, match_rate)
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+          JSON.stringify(audit.addedTxs), audit.matchRate]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save block audit into db. Reason: ` + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+
+  public async $getBlockPredictionsHistory(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
+
+      if (interval !== null) {
+        query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${div} ORDER BY height`;
+
+      const [rows] = await DB.query(query);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getPredictionsCount(): Promise<number> {
+    try {
+      const [rows] = await DB.query(`SELECT count(hash) as count FROM blocks_audits`);
+      return rows[0].count;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new BlocksAuditRepositories();
+

backend/src/repositories/BlocksRepository.ts

@@ -0,0 +1,633 @@
+import { BlockExtended } from '../mempool.interfaces';
+import DB from '../database';
+import logger from '../logger';
+import { Common } from '../api/common';
+import { prepareBlock } from '../utils/blocks-utils';
+import PoolsRepository from './PoolsRepository';
+import HashratesRepository from './HashratesRepository';
+import { escape } from 'mysql2';
+import BlocksSummariesRepository from './BlocksSummariesRepository';
+import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
+
+class BlocksRepository {
+  /**
+   * Save indexed block data in the database
+   */
+  public async $saveBlockInDatabase(block: BlockExtended) {
+    try {
+      const query = `INSERT INTO blocks(
+        height,           hash,                blockTimestamp, size,
+        weight,           tx_count,            coinbase_raw,   difficulty,
+        pool_id,          fees,                fee_span,       median_fee,
+        reward,           version,             bits,           nonce,
+        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+      ) VALUE (
+        ?, ?, FROM_UNIXTIME(?), ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?
+      )`;
+
+      const params: any[] = [
+        block.height,
+        block.id,
+        block.timestamp,
+        block.size,
+        block.weight,
+        block.tx_count,
+        block.extras.coinbaseRaw,
+        block.difficulty,
+        block.extras.pool?.id, // Should always be set to something
+        block.extras.totalFees,
+        JSON.stringify(block.extras.feeRange),
+        block.extras.medianFee,
+        block.extras.reward,
+        block.version,
+        block.bits,
+        block.nonce,
+        block.merkle_root,
+        block.previousblockhash,
+        block.extras.avgFee,
+        block.extras.avgFeeRate,
+      ];
+
+      await DB.query(query, params);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
+      } else {
+        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+
+  /**
+   * Get all block height that have not been indexed between [startHeight, endHeight]
+   */
+  public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
+    if (startHeight < endHeight) {
+      return [];
+    }
+
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT height
+        FROM blocks
+        WHERE height <= ? AND height >= ?
+        ORDER BY height DESC;
+      `, [startHeight, endHeight]);
+
+      const indexedBlockHeights: number[] = [];
+      rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
+      const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
+      const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
+
+      return missingBlocksHeights;
+    } catch (e) {
+      logger.err('Cannot retrieve blocks list to index. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get empty blocks for one or all pools
+   */
+  public async $countEmptyBlocks(poolId: number | null, interval: string | null = null): Promise<any> {
+    interval = Common.getSqlInterval(interval);
+
+    const params: any[] = [];
+    let query = `SELECT count(height) as count, pools.id as poolId
+      FROM blocks
+      JOIN pools on pools.id = blocks.pool_id
+      WHERE tx_count = 1`;
+
+    if (poolId) {
+      query += ` AND pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (interval) {
+      query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP by pools.id`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot count empty blocks. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Return most recent block height
+   */
+  public async $mostRecentBlockHeight(): Promise<number> {
+    try {
+      const [row] = await DB.query('SELECT MAX(height) as maxHeight from blocks');
+      return row[0]['maxHeight'];
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks count for a period
+   */
+  public async $blockCount(poolId: number | null, interval: string | null = null): Promise<number> {
+    interval = Common.getSqlInterval(interval);
+
+    const params: any[] = [];
+    let query = `SELECT count(height) as blockCount
+      FROM blocks`;
+
+    if (poolId) {
+      query += ` WHERE pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (interval) {
+      if (poolId) {
+        query += ` AND`;
+      } else {
+        query += ` WHERE`;
+      }
+      query += ` blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0].blockCount;
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks count between two dates
+   * @param poolId
+   * @param from - The oldest timestamp
+   * @param to - The newest timestamp
+   * @returns
+   */
+  public async $blockCountBetweenTimestamp(poolId: number | null, from: number, to: number): Promise<number> {
+    const params: any[] = [];
+    let query = `SELECT
+      count(height) as blockCount,
+      max(height) as lastBlockHeight
+      FROM blocks`;
+
+    if (poolId) {
+      query += ` WHERE pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (poolId) {
+      query += ` AND`;
+    } else {
+      query += ` WHERE`;
+    }
+    query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0];
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using timestamps). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks count for a period
+   */
+   public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
+    const params: any[] = [];
+    let query = `SELECT count(height) as blockCount
+      FROM blocks
+      WHERE height <= ${startHeight} AND height >= ${endHeight}`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0].blockCount;
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the oldest indexed block
+   */
+  public async $oldestBlockTimestamp(): Promise<number> {
+    const query = `SELECT UNIX_TIMESTAMP(blockTimestamp) as blockTimestamp
+      FROM blocks
+      ORDER BY height
+      LIMIT 1;`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+
+      if (rows.length <= 0) {
+        return -1;
+      }
+
+      return <number>rows[0].blockTimestamp;
+    } catch (e) {
+      logger.err('Cannot get oldest indexed block timestamp. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks mined by a specific mining pool
+   */
+  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
+    const pool = await PoolsRepository.$getPool(slug);
+    if (!pool) {
+      throw new Error('This mining pool does not exist ' + escape(slug));
+    }
+
+    const params: any[] = [];
+    let query = ` SELECT
+      height,
+      hash as id,
+      UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
+      size,
+      weight,
+      tx_count,
+      coinbase_raw,
+      difficulty,
+      fees,
+      fee_span,
+      median_fee,
+      reward,
+      version,
+      bits,
+      nonce,
+      merkle_root,
+      previous_block_hash as previousblockhash,
+      avg_fee,
+      avg_fee_rate
+      FROM blocks
+      WHERE pool_id = ?`;
+    params.push(pool.id);
+
+    if (startHeight !== undefined) {
+      query += ` AND height < ?`;
+      params.push(startHeight);
+    }
+
+    query += ` ORDER BY height DESC
+      LIMIT 10`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+
+      const blocks: BlockExtended[] = [];
+      for (const block of <object[]>rows) {
+        blocks.push(prepareBlock(block));
+      }
+
+      return blocks;
+    } catch (e) {
+      logger.err('Cannot get blocks for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get one block by height
+   */
+  public async $getBlockByHeight(height: number): Promise<object | null> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT
+        height,
+        hash,
+        hash as id,
+        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
+        size,
+        weight,
+        tx_count,
+        coinbase_raw,
+        difficulty,
+        pools.id as pool_id,
+        pools.name as pool_name,
+        pools.link as pool_link,
+        pools.slug as pool_slug,
+        pools.addresses as pool_addresses,
+        pools.regexes as pool_regexes,
+        fees,
+        fee_span,
+        median_fee,
+        reward,
+        version,
+        bits,
+        nonce,
+        merkle_root,
+        previous_block_hash as previousblockhash,
+        avg_fee,
+        avg_fee_rate
+        FROM blocks
+        JOIN pools ON blocks.pool_id = pools.id
+        WHERE height = ${height};
+      `);
+
+      if (rows.length <= 0) {
+        return null;
+      }
+
+      rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      return rows[0];
+    } catch (e) {
+      logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get one block by hash
+   */
+  public async $getBlockByHash(hash: string): Promise<object | null> {
+    try {
+      const query = `
+        SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
+        pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
+        pools.addresses as pool_addresses, pools.regexes as pool_regexes,
+        previous_block_hash as previousblockhash
+        FROM blocks
+        JOIN pools ON blocks.pool_id = pools.id
+        WHERE hash = '${hash}';
+      `;
+      const [rows]: any[] = await DB.query(query);
+
+      if (rows.length <= 0) {
+        return null;
+      }
+
+      rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      return rows[0];
+    } catch (e) {
+      logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Return blocks difficulty
+   */
+  public async $getBlocksDifficulty(): Promise<object[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get general block stats
+   */
+  public async $getBlockStats(blockCount: number): Promise<any> {
+    try {
+      // We need to use a subquery
+      const query = `
+        SELECT MIN(height) as startBlock, MAX(height) as endBlock, SUM(reward) as totalReward, SUM(fees) as totalFee, SUM(tx_count) as totalTx
+        FROM
+          (SELECT height, reward, fees, tx_count FROM blocks
+          ORDER by height DESC
+          LIMIT ?) as sub`;
+
+      const [rows]: any = await DB.query(query, [blockCount]);
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot generate reward stats. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Check if the chain of block hash is valid and delete data from the stale branch if needed
+   */
+  public async $validateChain(): Promise<boolean> {
+    try {
+      const start = new Date().getTime();
+      const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
+        UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
+
+      let partialMsg = false;
+      let idx = 1;
+      while (idx < blocks.length) {
+        if (blocks[idx].height - 1 !== blocks[idx - 1].height) {
+          if (partialMsg === false) {
+            logger.info('Some blocks are not indexed, skipping missing blocks during chain validation');
+            partialMsg = true;
+          }
+          ++idx;
+          continue;
+        }
+
+        if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
+          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
+          await this.$deleteBlocksFrom(blocks[idx - 1].height);
+          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
+          await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
+          await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
+          return false;
+        }
+        ++idx;
+      }
+
+      logger.debug(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`);
+      return true;
+    } catch (e) {
+      logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e));
+      return true; // Don't do anything if there is a db error
+    }
+  }
+
+  /**
+   * Delete blocks from the database from blockHeight
+   */
+  public async $deleteBlocksFrom(blockHeight: number) {
+    logger.info(`Delete newer blocks from height ${blockHeight} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Get the historical averaged block fees
+   */
+  public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(fees) as INT) as avgFees
+        FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block fees history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block rewards
+   */
+  public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(reward) as INT) as avgRewards
+        FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block rewards history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block fee rate percentiles
+   */
+   public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avgFee_0,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avgFee_10,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avgFee_25,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avgFee_50,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avgFee_75,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avgFee_90,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avgFee_100
+      FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block fee rates history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block sizes
+   */
+   public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(size) as INT) as avgSize
+      FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block weights
+   */
+   public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(weight) as INT) as avgWeight
+      FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get a list of blocks that have been indexed
+   */
+  public async $getIndexedBlocks(): Promise<any[]> {
+    try {
+      const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Return the oldest block  from a consecutive chain of block from the most recent one
+   */
+  public async $getOldestConsecutiveBlock(): Promise<any> {
+    try {
+      const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`);
+      for (let i = 0; i < rows.length - 1; ++i) {
+        if (rows[i].height - rows[i + 1].height > 1) {
+          return rows[i];
+        }
+      }
+      return rows[rows.length - 1];
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -0,0 +1,59 @@
+import DB from '../database';
+import logger from '../logger';
+import { BlockSummary } from '../mempool.interfaces';
+
+class BlocksSummariesRepository {
+  public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
+    try {
+      const [summary]: any[] = await DB.query(`SELECT * from blocks_summaries WHERE id = ?`, [id]);
+      if (summary.length > 0) {
+        summary[0].transactions = JSON.parse(summary[0].transactions);
+        return summary[0];
+      }
+    } catch (e) {
+      logger.err(`Cannot get block summary for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return undefined;
+  }
+
+  public async $saveSummary(height: number, summary: BlockSummary) {
+    try {
+      await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?)`, [height, summary.id, JSON.stringify(summary.transactions)]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block summary for ${summary.id} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block summary for ${summary.id}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getIndexedSummariesId(): Promise<string[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);
+      return rows.map(row => row.id);
+    } catch (e) {
+      logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return [];
+  }
+
+  /**
+   * Delete blocks from the database from blockHeight
+   */
+   public async $deleteBlocksFrom(blockHeight: number) {
+    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new BlocksSummariesRepository();
+

backend/src/repositories/DifficultyAdjustmentsRepository.ts

@@ -0,0 +1,124 @@
+import { Common } from '../api/common';
+import config from '../config';
+import DB from '../database';
+import logger from '../logger';
+import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
+
+class DifficultyAdjustmentsRepository {
+  public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise<void> {
+    if (adjustment.height === 1) {
+      return;
+    }
+
+    try {
+      const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`;
+      const params: any[] = [
+        adjustment.time,
+        adjustment.height,
+        adjustment.difficulty,
+        adjustment.adjustment,
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT 
+      CAST(AVG(UNIX_TIMESTAMP(time)) as INT) as time,
+      CAST(AVG(height) AS INT) as height,
+      CAST(AVG(difficulty) as DOUBLE) as difficulty,
+      CAST(AVG(adjustment) as DOUBLE) as adjustment
+      FROM difficulty_adjustments`;
+
+    if (interval) {
+      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${86400}`;
+
+    if (descOrder === true) {
+      query += ` ORDER BY height DESC`;
+    } else {
+      query += ` ORDER BY height`;
+    }
+
+    try {
+      const [rows] = await DB.query(query);
+      return rows as IndexedDifficultyAdjustment[];
+    } catch (e) {
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getRawAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT 
+      UNIX_TIMESTAMP(time) as time,
+      height as height,
+      difficulty as difficulty,
+      adjustment as adjustment
+      FROM difficulty_adjustments`;
+
+    if (interval) {
+      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    if (descOrder === true) {
+      query += ` ORDER BY height DESC`;
+    } else {
+      query += ` ORDER BY height`;
+    }
+
+    try {
+      const [rows] = await DB.query(query);
+      return rows as IndexedDifficultyAdjustment[];
+    } catch (e) {
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getAdjustmentsHeights(): Promise<number[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
+      return rows.map(block => block.height);
+    } catch (e: any) {
+      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
+    try {
+      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
+    } catch (e: any) {
+      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteLastAdjustment(): Promise<void> {
+    try {
+      logger.info(`Delete last difficulty adjustment from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
+    } catch (e: any) {
+      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+}
+
+export default new DifficultyAdjustmentsRepository();
+

backend/src/repositories/HashratesRepository.ts

@@ -0,0 +1,249 @@
+import { escape } from 'mysql2';
+import { Common } from '../api/common';
+import DB from '../database';
+import logger from '../logger';
+import PoolsRepository from './PoolsRepository';
+
+class HashratesRepository {
+  /**
+   * Save indexed block data in the database
+   */
+  public async $saveHashrates(hashrates: any) {
+    if (hashrates.length === 0) {
+      return;
+    }
+
+    let query = `INSERT INTO
+      hashrates(hashrate_timestamp, avg_hashrate, pool_id, share, type) VALUES`;
+
+    for (const hashrate of hashrates) {
+      query += ` (FROM_UNIXTIME(${hashrate.hashrateTimestamp}), ${hashrate.avgHashrate}, ${hashrate.poolId}, ${hashrate.share}, "${hashrate.type}"),`;
+    }
+    query = query.slice(0, -1);
+
+    try {
+      await DB.query(query);
+    } catch (e: any) {
+      logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getRawNetworkDailyHashrate(interval: string | null): Promise<any[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT
+      UNIX_TIMESTAMP(hashrate_timestamp) as timestamp,
+      avg_hashrate as avgHashrate
+      FROM hashrates`;
+
+    if (interval) {
+      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+        AND hashrates.type = 'daily'`;
+    } else {
+      query += ` WHERE hashrates.type = 'daily'`;
+    }
+
+    query += ` ORDER by hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT
+      CAST(AVG(UNIX_TIMESTAMP(hashrate_timestamp)) as INT) as timestamp,
+      CAST(AVG(avg_hashrate) as DOUBLE) as avgHashrate
+      FROM hashrates`;
+
+    if (interval) {
+      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+        AND hashrates.type = 'daily'`;
+    } else {
+      query += ` WHERE hashrates.type = 'daily'`;
+    }
+
+    query += ` GROUP BY UNIX_TIMESTAMP(hashrate_timestamp) DIV ${86400}`;
+    query += ` ORDER by hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
+    const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp
+      FROM hashrates
+      WHERE type = 'weekly'
+      GROUP BY hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows.map(row => row.timestamp);
+    } catch (e) {
+      logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Returns the current biggest pool hashrate history
+   */
+  public async $getPoolsWeeklyHashrate(interval: string | null): Promise<any[]> {
+    interval = Common.getSqlInterval(interval);
+
+    const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
+    if (topPoolsId.length === 0) {
+      return [];
+    }
+
+    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
+      FROM hashrates
+      JOIN pools on pools.id = pool_id`;
+
+    if (interval) {
+      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+        AND hashrates.type = 'weekly'
+        AND pool_id IN (${topPoolsId})`;
+    } else {
+      query += ` WHERE hashrates.type = 'weekly'
+        AND pool_id IN (${topPoolsId})`;
+    }
+
+    query += ` ORDER by hashrate_timestamp, FIELD(pool_id, ${topPoolsId})`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Returns a pool hashrate history
+   */
+  public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
+    const pool = await PoolsRepository.$getPool(slug);
+    if (!pool) {
+      throw new Error('This mining pool does not exist ' + escape(slug));
+    }
+
+    // Find hashrate boundaries
+    let query = `SELECT MIN(hashrate_timestamp) as firstTimestamp, MAX(hashrate_timestamp) as lastTimestamp
+      FROM hashrates
+      JOIN pools on pools.id = pool_id
+      WHERE hashrates.type = 'weekly' AND pool_id = ? AND avg_hashrate != 0
+      ORDER by hashrate_timestamp LIMIT 1`;
+
+    let boundaries = {
+      firstTimestamp: '1970-01-01',
+      lastTimestamp: '9999-01-01'
+    };
+
+    try {
+      const [rows]: any[] = await DB.query(query, [pool.id]);
+      boundaries = rows[0];
+    } catch (e) {
+      logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+
+    // Get hashrates entries between boundaries
+    query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
+      FROM hashrates
+      JOIN pools on pools.id = pool_id
+      WHERE hashrates.type = 'weekly' AND hashrate_timestamp BETWEEN ? AND ?
+      AND pool_id = ?
+      ORDER by hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Set latest run timestamp
+   */
+  public async $setLatestRun(key: string, val: number) {
+    const query = `UPDATE state SET number = ? WHERE name = ?`;
+
+    try {
+      await DB.query(query, [val, key]);
+    } catch (e) {
+      logger.err(`Cannot set last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get latest run timestamp
+   */
+  public async $getLatestRun(key: string): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [key]);
+
+      if (rows.length === 0) {
+        return 0;
+      }
+      return rows[0]['number'];
+    } catch (e) {
+      logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Delete most recent data points for re-indexing
+   */
+  public async $deleteLastEntries() {
+    logger.info(`Delete latest hashrates data points from the database`);
+
+    try {
+      const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
+      for (const row of rows) {
+        await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
+      }
+      // Re-run the hashrate indexing to fill up missing data
+      await this.$setLatestRun('last_hashrates_indexing', 0);
+      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+  
+  /**
+   * Delete hashrates from the database from timestamp
+   */
+  public async $deleteHashratesFromTimestamp(timestamp: number) {
+    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);
+      // Re-run the hashrate indexing to fill up missing data
+      await this.$setLatestRun('last_hashrates_indexing', 0);
+      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new HashratesRepository();

backend/src/repositories/PoolsRepository.ts

@@ -0,0 +1,99 @@
+import { Common } from '../api/common';
+import config from '../config';
+import DB from '../database';
+import logger from '../logger';
+import { PoolInfo, PoolTag } from '../mempool.interfaces';
+
+class PoolsRepository {
+  /**
+   * Get all pools tagging info
+   */
+  public async $getPools(): Promise<PoolTag[]> {
+    const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
+    return <PoolTag[]>rows;
+  }
+
+  /**
+   * Get unknown pool tagging info
+   */
+  public async $getUnknownPool(): Promise<PoolTag> {
+    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    return <PoolTag>rows[0];
+  }
+
+  /**
+   * Get basic pool info and block count
+   */
+  public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
+      FROM blocks
+      JOIN pools on pools.id = pool_id`;
+
+    if (interval) {
+      query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP BY pool_id
+      ORDER BY COUNT(height) DESC`;
+
+    try {
+      const [rows] = await DB.query(query);
+      return <PoolInfo[]>rows;
+    } catch (e) {
+      logger.err(`Cannot generate pools stats. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get basic pool info and block count between two timestamp
+   */
+   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+    const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
+      FROM pools
+      LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
+      GROUP BY pools.id`;
+
+    try {
+      const [rows] = await DB.query(query, [from, to]);
+      return <PoolInfo[]>rows;
+    } catch (e) {
+      logger.err('Cannot generate pools blocks count. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get mining pool statistics for one pool
+   */
+   public async $getPool(slug: string): Promise<PoolTag | null> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.slug = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [slug]);
+
+      if (rows.length < 1) {
+        return null;
+      }
+
+      rows[0].regexes = JSON.parse(rows[0].regexes);
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+        rows[0].addresses = []; // pools.json only contains mainnet addresses
+      } else {
+        rows[0].addresses = JSON.parse(rows[0].addresses);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new PoolsRepository();

backend/src/repositories/PricesRepository.ts

@@ -0,0 +1,42 @@
+import DB from '../database';
+import logger from '../logger';
+import { Prices } from '../tasks/price-updater';
+
+class PricesRepository {
+  public async $savePrices(time: number, prices: Prices): Promise<void> {
+    if (prices.USD === -1) {
+      // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
+      // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
+      return;
+    }
+
+    try {
+      await DB.query(`
+        INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
+        VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
+        [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestPriceTime(): Promise<number> {
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].time : 0;
+  }
+
+  public async $getLatestPriceTime(): Promise<number> {
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].time : 0;
+  }
+
+  public async $getPricesTimes(): Promise<number[]> {
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1`);
+    return times.map(time => time.time);
+  }
+}
+
+export default new PricesRepository();
+

backend/src/routes.ts

@@ -19,45 +19,63 @@ import loadingIndicators from './api/loading-indicators';
 import { Common } from './api/common';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import elementsParser from './api/liquid/elements-parser';
+import icons from './api/liquid/icons';
+import miningStats from './api/mining';
+import axios from 'axios';
+import mining from './api/mining';
+import BlocksRepository from './repositories/BlocksRepository';
+import HashratesRepository from './repositories/HashratesRepository';
+import difficultyAdjustment from './api/difficulty-adjustment';
+import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 class Routes {
   constructor() {}
 
-  public async get2HStatistics(req: Request, res: Response) {
-    const result = await statistics.$list2H();
-    res.json(result);
-  }
+  public async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y', req: Request, res: Response) {
+    res.header('Pragma', 'public');
+    res.header('Cache-control', 'public');
+    res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
 
-  public get24HStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['24h']);
-  }
-
-  public get1WHStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['1w']);
-  }
-
-  public get1MStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['1m']);
-  }
-
-  public get3MStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['3m']);
-  }
-
-  public get6MStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['6m']);
-  }
-
-  public get1YStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['1y']);
-  }
-
-  public get2YStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['2y']);
-  }
-  
-  public get3YStatistics(req: Request, res: Response) {
-    res.json(statistics.getCache()['3y']);
+    try {
+      let result;
+      switch (time as string) {
+        case '2h':
+          result = await statistics.$list2H();
+          res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+          break;
+        case '24h':
+          result = await statistics.$list24H();
+          res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+          break;
+        case '1w':
+          result = await statistics.$list1W();
+          break;
+        case '1m':
+          result = await statistics.$list1M();
+          break;
+        case '3m':
+          result = await statistics.$list3M();
+          break;
+        case '6m':
+          result = await statistics.$list6M();
+          break;
+        case '1y':
+          result = await statistics.$list1Y();
+          break;
+        case '2y':
+          result = await statistics.$list2Y();
+          break;
+        case '3y':
+          result = await statistics.$list3Y();
+          break;
+        default:
+          result = await statistics.$list2H();
+      }
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
   }
 
   public getInitData(req: Request, res: Response) {
@@ -69,7 +87,7 @@ class Routes {
     }
   }
 
-  public async getRecommendedFees(req: Request, res: Response) {
+  public getRecommendedFees(req: Request, res: Response) {
     if (!mempool.isInSync()) {
       res.statusCode = 503;
       res.send('Service Unavailable');
@@ -104,6 +122,30 @@ class Routes {
     res.json(times);
   }
 
+  public async $getBatchedOutspends(req: Request, res: Response) {
+    if (!Array.isArray(req.query.txId)) {
+      res.status(500).send('Not an array');
+      return;
+    }
+    if (req.query.txId.length > 50) {
+      res.status(400).send('Too many txids requested');
+      return;
+    }
+    const txIds: string[] = [];
+    for (const _txId in req.query.txId) {
+      if (typeof req.query.txId[_txId] === 'string') {
+        txIds.push(req.query.txId[_txId].toString());
+      }
+    }
+
+    try {
+      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds);
+      res.json(batchedOutspends);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public getCpfpInfo(req: Request, res: Response) {
     if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
       res.status(501).send(`Invalid transaction ID.`);
@@ -521,10 +563,206 @@ class Routes {
     }
   }
 
+  public async $getPool(req: Request, res: Response) {
+    try {
+      const stats = await mining.$getPoolStat(req.params.slug);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  public async $getPoolBlocks(req: Request, res: Response) {
+    try {
+      const poolBlocks = await BlocksRepository.$getBlocksByPool(
+        req.params.slug,
+        req.params.height === undefined ? undefined : parseInt(req.params.height, 10),
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(poolBlocks);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  public async $getPools(req: Request, res: Response) {
+    try {
+      const stats = await miningStats.$getPoolsStats(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getPoolsHistoricalHashrate(req: Request, res: Response) {
+    try {
+      const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(hashrates);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getPoolHistoricalHashrate(req: Request, res: Response) {
+    try {
+      const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(hashrates);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  public async $getHistoricalHashrate(req: Request, res: Response) {
+    let currentHashrate = 0, currentDifficulty = 0;
+    try {
+      currentHashrate = await bitcoinClient.getNetworkHashPs();
+      currentDifficulty = await bitcoinClient.getDifficulty();
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
+    }
+
+    try {
+      const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
+      const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json({
+        hashrates: hashrates,
+        difficulty: difficulty,
+        currentHashrate: currentHashrate,
+        currentDifficulty: currentDifficulty,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getHistoricalBlockFees(req: Request, res: Response) {
+    try {
+      const blockFees = await mining.$getHistoricalBlockFees(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockFees);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getHistoricalBlockRewards(req: Request, res: Response) {
+    try {
+      const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockRewards);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getHistoricalBlockFeeRates(req: Request, res: Response) {
+    try {
+      const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockFeeRates);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) {
+    try {
+      const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval);
+      const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        sizes: blockSizes,
+        weights: blockWeights
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getDifficultyAdjustments(req: Request, res: Response) {
+    try {
+      const difficulty = await DifficultyAdjustmentsRepository.$getRawAdjustments(req.params.interval, true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getHistoricalBlockPrediction(req: Request, res: Response) {
+    try {
+      const blockPredictions = await mining.$getBlockPredictionsHistory(req.params.interval);
+      const blockCount = await BlocksAuditsRepository.$getPredictionsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockPredictions.map(prediction => [prediction.time, prediction.height, prediction.match_rate]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlock(req: Request, res: Response) {
     try {
-      const result = await bitcoinApi.$getBlock(req.params.hash);
-      res.json(result);
+      const block = await blocks.$getBlock(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(block);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
@@ -540,10 +778,32 @@ class Routes {
     }
   }
 
+  public async getStrippedBlockTransactions(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlocks(req: Request, res: Response) {
     try {
-      loadingIndicators.setProgress('blocks', 0);
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+        const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+        res.json(await blocks.$getBlocks(height, 15));
+      } else { // Liquid, Bisq
+        return await this.getLegacyBlocks(req, res);
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 
+  public async getLegacyBlocks(req: Request, res: Response) {
+    try {
       const returnBlocks: IEsploraApi.Block[] = [];
       const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();
 
@@ -557,7 +817,7 @@ class Routes {
       }
 
       let nextHash = startFromHash;
-      for (let i = 0; i < 10; i++) {
+      for (let i = 0; i < 10 && nextHash; i++) {
         const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
         if (localBlock) {
           returnBlocks.push(localBlock);
@@ -567,16 +827,15 @@ class Routes {
           returnBlocks.push(block);
           nextHash = block.previousblockhash;
         }
-        loadingIndicators.setProgress('blocks', i / 10 * 100);
       }
 
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(returnBlocks);
     } catch (e) {
-      loadingIndicators.setProgress('blocks', 100);
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
-
+  
   public async getBlockTransactions(req: Request, res: Response) {
     try {
       loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
@@ -588,9 +847,9 @@ class Routes {
       const endIndex = Math.min(startingIndex + 10, txIds.length);
       for (let i = startingIndex; i < endIndex; i++) {
         try {
-          const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true);
+          const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true);
           transactions.push(transaction);
-          loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i + 1) / endIndex * 100);
+          loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100);
         } catch (e) {
           logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e));
         }
@@ -667,7 +926,13 @@ class Routes {
   }
 
   public async getMempool(req: Request, res: Response) {
-    res.status(501).send('Not implemented');
+    const info = mempool.getMempoolInfo();
+    res.json({
+      count: info.size,
+      vsize: info.bytes,
+      total_fee: info.total_fee * 1e8,
+      fee_histogram: []
+    });
   }
 
   public async getMempoolTxIds(req: Request, res: Response) {
@@ -688,6 +953,16 @@ class Routes {
     }
   }
 
+  public async getBlockTipHash(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getBlockHashTip();
+      res.setHeader('content-type', 'text/plain');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getTxIdsForBlock(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
@@ -706,61 +981,18 @@ class Routes {
     }
   }
 
-  public getTransactionOutspends(req: Request, res: Response) {
-    res.status(501).send('Not implemented');
+  public async getTransactionOutspends(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getOutspends(req.params.txId);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
   }
 
   public getDifficultyChange(req: Request, res: Response) {
     try {
-      const DATime = blocks.getLastDifficultyAdjustmentTime();
-      const previousRetarget = blocks.getPreviousDifficultyRetarget();
-      const blockHeight = blocks.getCurrentBlockHeight();
-
-      const now = new Date().getTime() / 1000;
-      const diff = now - DATime;
-      const blocksInEpoch = blockHeight % 2016;
-      const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
-      const remainingBlocks = 2016 - blocksInEpoch;
-      const nextRetargetHeight = blockHeight + remainingBlocks;
-
-      let difficultyChange = 0;
-      if (remainingBlocks < 1870) {
-        if (blocksInEpoch > 0) {
-          difficultyChange = (600 / (diff / blocksInEpoch ) - 1) * 100;
-        }
-        if (difficultyChange > 300) {
-          difficultyChange = 300;
-        }
-        if (difficultyChange < -75) {
-          difficultyChange = -75;
-        }
-      }
-
-      const timeAvgDiff = difficultyChange * 0.1;
-
-      let timeAvgMins = 10;
-      if (timeAvgDiff > 0) {
-        timeAvgMins -= Math.abs(timeAvgDiff);
-      } else {
-        timeAvgMins += Math.abs(timeAvgDiff);
-      }
-
-      const timeAvg = timeAvgMins * 60;
-      const remainingTime =  remainingBlocks * timeAvg;
-      const estimatedRetargetDate = remainingTime + now;
-
-      const result = {
-        progressPercent,
-        difficultyChange,
-        estimatedRetargetDate,
-        remainingBlocks,
-        remainingTime,
-        previousRetarget,
-        nextRetargetHeight,
-        timeAvg,
-      };
-      res.json(result);
-
+      res.json(difficultyAdjustment.getDifficultyAdjustment());
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
@@ -807,6 +1039,55 @@ class Routes {
         : (e.message || 'Error'));
     }
   }
+
+  public getLiquidIcon(req: Request, res: Response) {
+    const result = icons.getIconByAssetId(req.params.assetId);
+    if (result) {
+      res.setHeader('content-type', 'image/png');
+      res.setHeader('content-length', result.length);
+      res.send(result);
+    } else {
+      res.status(404).send('Asset icon not found');
+    }
+  }
+
+  public getAllLiquidIcon(req: Request, res: Response) {
+    const result = icons.getAllIconIds();
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Asset icons not found');
+    }
+  }
+
+  public async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  public async $getAssetGroup(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
+        { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  public async $getRewardStats(req: Request, res: Response) {
+    try {
+      const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10));
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(response);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
 }
 
 export default new Routes();

backend/src/rpc-api/commands.ts

@@ -0,0 +1,92 @@
+module.exports = {
+  addMultiSigAddress: 'addmultisigaddress',
+  addNode: 'addnode', // bitcoind v0.8.0+
+  backupWallet: 'backupwallet',
+  createMultiSig: 'createmultisig',
+  createRawTransaction: 'createrawtransaction', // bitcoind v0.7.0+
+  decodeRawTransaction: 'decoderawtransaction', // bitcoind v0.7.0+
+  decodeScript: 'decodescript',
+  dumpPrivKey: 'dumpprivkey',
+  dumpWallet: 'dumpwallet', // bitcoind v0.9.0+
+  encryptWallet: 'encryptwallet',
+  estimateFee: 'estimatefee', // bitcoind v0.10.0x
+  estimatePriority: 'estimatepriority', // bitcoind v0.10.0+
+  generate: 'generate', // bitcoind v0.11.0+
+  getAccount: 'getaccount',
+  getAccountAddress: 'getaccountaddress',
+  getAddedNodeInfo: 'getaddednodeinfo', // bitcoind v0.8.0+
+  getAddressesByAccount: 'getaddressesbyaccount',
+  getBalance: 'getbalance',
+  getBestBlockHash: 'getbestblockhash', // bitcoind v0.9.0+
+  getBlock: 'getblock',
+  getBlockStats: 'getblockstats',
+  getBlockFilter: 'getblockfilter',
+  getBlockchainInfo: 'getblockchaininfo', // bitcoind v0.9.2+
+  getBlockCount: 'getblockcount',
+  getBlockHash: 'getblockhash',
+  getBlockHeader: 'getblockheader',
+  getBlockTemplate: 'getblocktemplate', // bitcoind v0.7.0+
+  getChainTips: 'getchaintips', // bitcoind v0.10.0+
+  getChainTxStats: 'getchaintxstats',
+  getConnectionCount: 'getconnectioncount',
+  getDifficulty: 'getdifficulty',
+  getGenerate: 'getgenerate',
+  getInfo: 'getinfo',
+  getMempoolAncestors: 'getmempoolancestors',
+  getMempoolDescendants: 'getmempooldescendants',
+  getMempoolEntry: 'getmempoolentry',
+  getMempoolInfo: 'getmempoolinfo', // bitcoind v0.10+
+  getMiningInfo: 'getmininginfo',
+  getNetTotals: 'getnettotals',
+  getNetworkInfo: 'getnetworkinfo', // bitcoind v0.9.2+
+  getNetworkHashPs: 'getnetworkhashps', // bitcoind v0.9.0+
+  getNewAddress: 'getnewaddress',
+  getPeerInfo: 'getpeerinfo', // bitcoind v0.7.0+
+  getRawChangeAddress: 'getrawchangeaddress', // bitcoin v0.9+
+  getRawMemPool: 'getrawmempool', // bitcoind v0.7.0+
+  getRawTransaction: 'getrawtransaction', // bitcoind v0.7.0+
+  getReceivedByAccount: 'getreceivedbyaccount',
+  getReceivedByAddress: 'getreceivedbyaddress',
+  getTransaction: 'gettransaction',
+  getTxOut: 'gettxout', // bitcoind v0.7.0+
+  getTxOutProof: 'gettxoutproof', // bitcoind v0.11.0+
+  getTxOutSetInfo: 'gettxoutsetinfo', // bitcoind v0.7.0+
+  getUnconfirmedBalance: 'getunconfirmedbalance', // bitcoind v0.9.0+
+  getWalletInfo: 'getwalletinfo', // bitcoind v0.9.2+
+  help: 'help',
+  importAddress: 'importaddress', // bitcoind v0.10.0+
+  importPrivKey: 'importprivkey',
+  importWallet: 'importwallet', // bitcoind v0.9.0+
+  keypoolRefill: 'keypoolrefill',
+  keyPoolRefill: 'keypoolrefill',
+  listAccounts: 'listaccounts',
+  listAddressGroupings: 'listaddressgroupings', // bitcoind v0.7.0+
+  listLockUnspent: 'listlockunspent', // bitcoind v0.8.0+
+  listReceivedByAccount: 'listreceivedbyaccount',
+  listReceivedByAddress: 'listreceivedbyaddress',
+  listSinceBlock: 'listsinceblock',
+  listTransactions: 'listtransactions',
+  listUnspent: 'listunspent', // bitcoind v0.7.0+
+  lockUnspent: 'lockunspent', // bitcoind v0.8.0+
+  move: 'move',
+  ping: 'ping', // bitcoind v0.9.0+
+  prioritiseTransaction: 'prioritisetransaction', // bitcoind v0.10.0+
+  sendFrom: 'sendfrom',
+  sendMany: 'sendmany',
+  sendRawTransaction: 'sendrawtransaction', // bitcoind v0.7.0+
+  sendToAddress: 'sendtoaddress',
+  setAccount: 'setaccount',
+  setGenerate: 'setgenerate',
+  setTxFee: 'settxfee',
+  signMessage: 'signmessage',
+  signRawTransaction: 'signrawtransaction', // bitcoind v0.7.0+
+  stop: 'stop',
+  submitBlock: 'submitblock', // bitcoind v0.7.0+
+  validateAddress: 'validateaddress',
+  verifyChain: 'verifychain', // bitcoind v0.9.0+
+  verifyMessage: 'verifymessage',
+  verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
+  walletLock: 'walletlock',
+  walletPassphrase: 'walletpassphrase',
+  walletPassphraseChange: 'walletpassphrasechange'
+}

backend/src/rpc-api/index.ts

@@ -0,0 +1,61 @@
+var commands = require('./commands')
+var rpc = require('./jsonrpc')
+
+// ===----------------------------------------------------------------------===//
+// JsonRPC
+// ===----------------------------------------------------------------------===//
+function Client (opts) {
+  // @ts-ignore
+  this.rpc = new rpc.JsonRPC(opts)
+}
+
+// ===----------------------------------------------------------------------===//
+// cmd
+// ===----------------------------------------------------------------------===//
+Client.prototype.cmd = function () {
+  var args = [].slice.call(arguments)
+  var cmd = args.shift()
+
+  callRpc(cmd, args, this.rpc)
+}
+
+// ===----------------------------------------------------------------------===//
+// callRpc
+// ===----------------------------------------------------------------------===//
+function callRpc (cmd, args, rpc) {
+  var fn = args[args.length - 1]
+
+  // If the last argument is a callback, pop it from the args list
+  if (typeof fn === 'function') {
+    args.pop()
+  } else {
+    fn = function () {}
+  }
+
+  return rpc.call(cmd, args, function () {
+    var args = [].slice.call(arguments)
+      // @ts-ignore
+    args.unshift(null)
+      // @ts-ignore
+    fn.apply(this, args)
+  }, function (err) {
+    fn(err)
+  })
+}
+
+// ===----------------------------------------------------------------------===//
+// Initialize wrappers
+// ===----------------------------------------------------------------------===//
+(function () {
+  for (var protoFn in commands) {
+    (function (protoFn) {
+      Client.prototype[protoFn] = function () {
+        var args = [].slice.call(arguments)
+        return callRpc(commands[protoFn], args, this.rpc)
+      }
+    })(protoFn)
+  }
+})()
+
+// Export!
+module.exports.Client = Client;

backend/src/rpc-api/jsonrpc.ts

@@ -0,0 +1,162 @@
+var http = require('http')
+var https = require('https')
+
+var JsonRPC = function (opts) {
+  // @ts-ignore
+  this.opts = opts || {}
+  // @ts-ignore
+  this.http = this.opts.ssl ? https : http
+}
+
+JsonRPC.prototype.call = function (method, params) {
+  return new Promise((resolve, reject) => {
+    var time = Date.now()
+    var requestJSON
+
+    if (Array.isArray(method)) {
+      // multiple rpc batch call
+      requestJSON = []
+      method.forEach(function (batchCall, i) {
+        requestJSON.push({
+          id: time + '-' + i,
+          method: batchCall.method,
+          params: batchCall.params
+        })
+      })
+    } else {
+      // single rpc call
+      requestJSON = {
+        id: time,
+        method: method,
+        params: params
+      }
+    }
+
+    // First we encode the request into JSON
+    requestJSON = JSON.stringify(requestJSON)
+
+    // prepare request options
+    var requestOptions = {
+      host: this.opts.host || 'localhost',
+      port: this.opts.port || 8332,
+      method: 'POST',
+      path: '/',
+      headers: {
+        'Host': this.opts.host || 'localhost',
+        'Content-Length': requestJSON.length
+      },
+      agent: false,
+      rejectUnauthorized: this.opts.ssl && this.opts.sslStrict !== false
+    }
+
+    if (this.opts.ssl && this.opts.sslCa) {
+    // @ts-ignore 
+      requestOptions.ca = this.opts.sslCa
+    }
+
+    // use HTTP auth if user and password set
+    if (this.opts.user && this.opts.pass) {
+      // @ts-ignore
+      requestOptions.auth = this.opts.user + ':' + this.opts.pass
+    }
+
+    // Now we'll make a request to the server
+    var cbCalled = false
+    var request = this.http.request(requestOptions)
+
+    // start request timeout timer
+    var reqTimeout = setTimeout(function () {
+      if (cbCalled) return
+      cbCalled = true
+      request.abort()
+      var err = new Error('ETIMEDOUT')
+      // @ts-ignore
+      err.code = 'ETIMEDOUT'
+      reject(err)
+    }, this.opts.timeout || 30000)
+
+    // set additional timeout on socket in case of remote freeze after sending headers
+    request.setTimeout(this.opts.timeout || 30000, function () {
+      if (cbCalled) return
+      cbCalled = true
+      request.abort()
+      var err = new Error('ESOCKETTIMEDOUT')
+      // @ts-ignore
+      err.code = 'ESOCKETTIMEDOUT'
+      reject(err)
+    })
+
+    request.on('error', function (err) {
+      if (cbCalled) return
+      cbCalled = true
+      clearTimeout(reqTimeout)
+      reject(err)
+    })
+
+    request.on('response', function (response) {
+      clearTimeout(reqTimeout)
+
+      // We need to buffer the response chunks in a nonblocking way.
+      var buffer = ''
+      response.on('data', function (chunk) {
+        buffer = buffer + chunk
+      })
+      // When all the responses are finished, we decode the JSON and
+      // depending on whether it's got a result or an error, we call
+      // emitSuccess or emitError on the promise.
+      response.on('end', function () {
+        var err
+
+        if (cbCalled) return
+        cbCalled = true
+
+        try {
+          var decoded = JSON.parse(buffer)
+        } catch (e) {
+          if (response.statusCode !== 200) {
+            err = new Error('Invalid params, response status code: ' + response.statusCode)
+            err.code = -32602
+            reject(err)
+          } else {
+            err = new Error('Problem parsing JSON response from server')
+            err.code = -32603
+            reject(err)
+          }
+          return
+        }
+
+        if (!Array.isArray(decoded)) {
+          decoded = [decoded]
+        }
+
+        // iterate over each response, normally there will be just one
+        // unless a batch rpc call response is being processed
+        decoded.forEach(function (decodedResponse, i) {
+          if (decodedResponse.hasOwnProperty('error') && decodedResponse.error != null) {
+            if (reject) {
+              err = new Error(decodedResponse.error.message || '')
+              if (decodedResponse.error.code) {
+                err.code = decodedResponse.error.code
+              }
+              reject(err)
+            }
+          } else if (decodedResponse.hasOwnProperty('result')) {
+            // @ts-ignore
+            resolve(decodedResponse.result, response.headers)
+          } else {
+            if (reject) {
+              err = new Error(decodedResponse.error.message || '')
+              if (decodedResponse.error.code) {
+                err.code = decodedResponse.error.code
+              }
+              reject(err)
+            }
+          }
+        })
+      })
+    })
+    request.end(requestJSON);
+  });
+}
+
+module.exports.JsonRPC = JsonRPC

backend/src/sync-assets.ts

@@ -0,0 +1,85 @@
+import axios, { AxiosResponse } from 'axios';
+import * as fs from 'fs';
+import config from './config';
+import backendInfo from './api/backend-info';
+import logger from './logger';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+
+const PATH = './';
+
+class SyncAssets {
+  constructor() { }
+
+  public async syncAssets$() {
+    for (const url of config.MEMPOOL.EXTERNAL_ASSETS) {
+      try {
+        await this.downloadFile$(url);
+      } catch (e) {
+        throw new Error(`Failed to download external asset. ` + (e instanceof Error ? e.message : e));
+      }
+    }
+  }
+
+  private async downloadFile$(url: string) {
+    return new Promise((resolve, reject) => {
+      const fileName = url.split('/').slice(-1)[0];
+
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          }
+
+          const agent = new SocksProxyAgent(socksOptions);
+
+          logger.info(`Downloading external asset ${fileName} over the Tor network...`);
+          return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
+            httpAgent: agent,
+            httpsAgent: agent,
+            responseType: 'stream',
+            timeout: 30000
+          }).then(function (response) {
+            const writer = fs.createWriteStream(PATH + fileName);
+            writer.on('finish', () => {
+              logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
+              resolve(0);
+            });
+            response.data.pipe(writer);
+          });
+        } else {
+          logger.info(`Downloading external asset ${fileName} over clearnet...`);
+          return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
+            responseType: 'stream',
+            timeout: 30000
+          }).then(function (response) {
+            const writer = fs.createWriteStream(PATH + fileName);
+            writer.on('finish', () => {
+              logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
+              resolve(0);
+            });
+            response.data.pipe(writer);
+          });
+        }
+      } catch (e: any) {
+        reject(e);
+      }
+    });
+  }
+}
+
+export default new SyncAssets();

backend/src/tasks/pools-updater.ts

@@ -0,0 +1,180 @@
+import axios, { AxiosResponse } from 'axios';
+import poolsParser from '../api/pools-parser';
+import config from '../config';
+import DB from '../database';
+import backendInfo from '../api/backend-info';
+import logger from '../logger';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import * as https from 'https';
+
+/**
+ * Maintain the most recent version of pools.json
+ */
+class PoolsUpdater {
+  lastRun: number = 0;
+  currentSha: any = undefined;
+  poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
+  treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
+
+  constructor() {
+  }
+
+  public async updatePoolsJson() {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return;
+    }
+
+    const oneWeek = 604800;
+    const oneDay = 86400;
+
+    const now = new Date().getTime() / 1000;
+    if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart
+      return;
+    }
+
+    this.lastRun = now;
+
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`);
+    } else {
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`);
+    }
+
+    try {
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
+      if (githubSha === undefined) {
+        return;
+      }
+
+      if (config.DATABASE.ENABLED === true) {
+        this.currentSha = await this.getShaFromDb();
+      }
+
+      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      if (this.currentSha !== undefined && this.currentSha === githubSha) {
+        return;
+      }
+
+      if (this.currentSha === undefined) {
+        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`);
+      } else {
+        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`);
+      }
+      const poolsJson = await this.query(this.poolsUrl);
+      if (poolsJson === undefined) {
+        return;
+      }
+      await poolsParser.migratePoolsJson(poolsJson);
+      await this.updateDBSha(githubSha);
+      logger.notice('PoolsUpdater completed');
+
+    } catch (e) {
+      this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
+      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Fetch our latest pools.json sha from the db
+   */
+  private async updateDBSha(githubSha: string) {
+    this.currentSha = githubSha;
+    if (config.DATABASE.ENABLED === true) {
+      try {
+        await DB.query('DELETE FROM state where name="pools_json_sha"');
+        await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
+      } catch (e) {
+        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
+      }
+    }
+  }
+
+  /**
+   * Fetch our latest pools.json sha from the db
+   */
+  private async getShaFromDb(): Promise<string | undefined> {
+    try {
+      const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
+      return (rows.length > 0 ? rows[0].string : undefined);
+    } catch (e) {
+      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
+      return undefined;
+    }
+  }
+
+  /**
+   * Fetch our latest pools.json sha from github
+   */
+  private async fetchPoolsSha(): Promise<string | undefined> {
+    const response = await this.query(this.treeUrl);
+
+    if (response !== undefined) {
+      for (const file of response['tree']) {
+        if (file['path'] === 'pools.json') {
+          return file['sha'];
+        }
+      }
+    }
+
+    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`);
+    return undefined;
+  }
+
+  /**
+   * Http request wrapper
+   */
+  private async query(path): Promise<object | undefined> {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpsAgent?: https.Agent;
+    };
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+    let retry = 0;
+
+    while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+        }
+
+        const data: AxiosResponse = await axios.get(path, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
+          throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
+        }
+        return data.data;
+      } catch (e) {
+        logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
+        retry++;
+      }
+      await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+    }
+    return undefined;
+  }
+}
+
+export default new PoolsUpdater();

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class BitfinexApi implements PriceFeed {
+  public name: string = 'Bitfinex';
+  public currencies: string[] = ['USD', 'EUR', 'GPB', 'JPY'];
+
+  public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
+  public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['last_price'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '1h').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price[0] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price[2];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default BitfinexApi;

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -0,0 +1,24 @@
+import { query } from '../../utils/axios-query';
+import { PriceFeed, PriceHistory } from '../price-updater';
+
+class BitflyerApi implements PriceFeed {
+  public name: string = 'Bitflyer';
+  public currencies: string[] = ['USD', 'EUR', 'JPY'];
+
+  public url: string = 'https://api.bitflyer.com/v1/ticker?product_code=BTC_';
+  public urlHist: string = '';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['ltp'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    return [];
+  }
+}
+
+export default BitflyerApi;

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -0,0 +1,42 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class CoinbaseApi implements PriceFeed {
+  public name: string = 'Coinbase';
+  public currencies: string[] = ['USD', 'EUR', 'GBP'];
+
+  public url: string = 'https://api.coinbase.com/v2/prices/spot?currency=';
+  public urlHist: string = 'https://api.exchange.coinbase.com/products/BTC-{CURRENCY}/candles?granularity={GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['data']['amount'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default CoinbaseApi;

backend/src/tasks/price-feeds/ftx-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class FtxApi implements PriceFeed {
+  public name: string = 'FTX';
+  public currencies: string[] = ['USD', 'BRZ', 'EUR', 'JPY', 'AUD'];
+
+  public url: string = 'https://ftx.com/api/markets/BTC/';
+  public urlHist: string = 'https://ftx.com/api/markets/BTC/{CURRENCY}/candles?resolution={GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['result']['last'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response['result'] : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price['time'] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price['close'];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default FtxApi;

backend/src/tasks/price-feeds/gemini-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class GeminiApi implements PriceFeed {
+  public name: string = 'Gemini';
+  public currencies: string[] = ['USD', 'EUR', 'GBP', 'SGD'];
+
+  public url: string = 'https://api.gemini.com/v1/pubticker/BTC';
+  public urlHist: string = 'https://api.gemini.com/v2/candles/BTC{CURRENCY}/{GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['last'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '1hr').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price[0] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default GeminiApi;

backend/src/tasks/price-feeds/kraken-api.ts

@@ -0,0 +1,95 @@
+import logger from '../../logger';
+import PricesRepository from '../../repositories/PricesRepository';
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class KrakenApi implements PriceFeed {
+  public name: string = 'Kraken';
+  public currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+
+  public url: string = 'https://api.kraken.com/0/public/Ticker?pair=XBT';
+  public urlHist: string = 'https://api.kraken.com/0/public/OHLC?interval={GRANULARITY}&pair=XBT';
+
+  constructor() {
+  }
+
+  private getTicker(currency) {
+    let ticker = `XXBTZ${currency}`;
+    if (['CHF', 'AUD'].includes(currency)) {
+      ticker = `XBT${currency}`;
+    }
+    return ticker;
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '60') + currency);
+      const pricesRaw = response ? response['result'][this.getTicker(currency)] : [];
+
+      for (const price of pricesRaw) {
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+
+  /**
+   * Fetch weekly price and save it into the database
+   */
+  public async $insertHistoricalPrice(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    // EUR weekly price history goes back to timestamp 1378339200 (September 5, 2013)
+    // USD weekly price history goes back to timestamp 1380758400 (October 3, 2013)
+    // GBP weekly price history goes back to timestamp 1415232000 (November 6, 2014)
+    // JPY weekly price history goes back to timestamp 1415232000 (November 6, 2014)
+    // CAD weekly price history goes back to timestamp 1436400000 (July 9, 2015)
+    // CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019)
+    // AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020)
+
+    const priceHistory: any = {}; // map: timestamp -> Prices
+
+    for (const currency of this.currencies) {
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency);
+      const priceHistoryRaw = response ? response['result'][this.getTicker(currency)] : [];
+
+      for (const price of priceHistoryRaw) {
+        if (existingPriceTimes.includes(parseInt(price[0]))) {
+          continue;
+        }
+
+        // prices[0] = kraken price timestamp
+        // prices[4] = closing price
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    for (const time in priceHistory) {
+      await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]);
+    }
+
+    if (Object.keys(priceHistory).length > 0) {
+      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
+    }
+  }
+}
+
+export default KrakenApi;

backend/src/tasks/price-feeds/mtgox-weekly.json

@@ -0,0 +1,762 @@
+[
+  {
+    "ct": 1279497600,
+    "c": "0.08584"
+  },
+  {
+    "ct": 1280102400,
+    "c": "0.0505"
+  },
+  {
+    "ct": 1280707200,
+    "c": "0.0611"
+  },
+  {
+    "ct": 1281312000,
+    "c": "0.0609"
+  },
+  {
+    "ct": 1281916800,
+    "c": "0.06529"
+  },
+  {
+    "ct": 1282521600,
+    "c": "0.066"
+  },
+  {
+    "ct": 1283126400,
+    "c": "0.064"
+  },
+  {
+    "ct": 1283731200,
+    "c": "0.06165"
+  },
+  {
+    "ct": 1284336000,
+    "c": "0.0615"
+  },
+  {
+    "ct": 1284940800,
+    "c": "0.0627"
+  },
+  {
+    "ct": 1285545600,
+    "c": "0.0622"
+  },
+  {
+    "ct": 1286150400,
+    "c": "0.06111"
+  },
+  {
+    "ct": 1286755200,
+    "c": "0.0965"
+  },
+  {
+    "ct": 1287360000,
+    "c": "0.102"
+  },
+  {
+    "ct": 1287964800,
+    "c": "0.11501"
+  },
+  {
+    "ct": 1288569600,
+    "c": "0.1925"
+  },
+  {
+    "ct": 1289174400,
+    "c": "0.34"
+  },
+  {
+    "ct": 1289779200,
+    "c": "0.27904"
+  },
+  {
+    "ct": 1290384000,
+    "c": "0.27675"
+  },
+  {
+    "ct": 1290988800,
+    "c": "0.27"
+  },
+  {
+    "ct": 1291593600,
+    "c": "0.19"
+  },
+  {
+    "ct": 1292198400,
+    "c": "0.2189"
+  },
+  {
+    "ct": 1292803200,
+    "c": "0.2401"
+  },
+  {
+    "ct": 1293408000,
+    "c": "0.263"
+  },
+  {
+    "ct": 1294012800,
+    "c": "0.29997"
+  },
+  {
+    "ct": 1294617600,
+    "c": "0.323"
+  },
+  {
+    "ct": 1295222400,
+    "c": "0.38679"
+  },
+  {
+    "ct": 1295827200,
+    "c": "0.4424"
+  },
+  {
+    "ct": 1296432000,
+    "c": "0.4799"
+  },
+  {
+    "ct": 1297036800,
+    "c": "0.8968"
+  },
+  {
+    "ct": 1297641600,
+    "c": "1.05"
+  },
+  {
+    "ct": 1298246400,
+    "c": "0.865"
+  },
+  {
+    "ct": 1298851200,
+    "c": "0.89"
+  },
+  {
+    "ct": 1299456000,
+    "c": "0.8999"
+  },
+  {
+    "ct": 1300060800,
+    "c": "0.89249"
+  },
+  {
+    "ct": 1300665600,
+    "c": "0.75218"
+  },
+  {
+    "ct": 1301270400,
+    "c": "0.82754"
+  },
+  {
+    "ct": 1301875200,
+    "c": "0.779"
+  },
+  {
+    "ct": 1302480000,
+    "c": "0.7369"
+  },
+  {
+    "ct": 1303084800,
+    "c": "1.1123"
+  },
+  {
+    "ct": 1303689600,
+    "c": "1.6311"
+  },
+  {
+    "ct": 1304294400,
+    "c": "3.03311"
+  },
+  {
+    "ct": 1304899200,
+    "c": "3.8659"
+  },
+  {
+    "ct": 1305504000,
+    "c": "6.98701"
+  },
+  {
+    "ct": 1306108800,
+    "c": "6.6901"
+  },
+  {
+    "ct": 1306713600,
+    "c": "8.4"
+  },
+  {
+    "ct": 1307318400,
+    "c": "16.7"
+  },
+  {
+    "ct": 1307923200,
+    "c": "18.5464"
+  },
+  {
+    "ct": 1308528000,
+    "c": "17.51"
+  },
+  {
+    "ct": 1309132800,
+    "c": "16.45001"
+  },
+  {
+    "ct": 1309737600,
+    "c": "15.44049"
+  },
+  {
+    "ct": 1310342400,
+    "c": "14.879"
+  },
+  {
+    "ct": 1310947200,
+    "c": "13.16"
+  },
+  {
+    "ct": 1311552000,
+    "c": "13.98001"
+  },
+  {
+    "ct": 1312156800,
+    "c": "13.35"
+  },
+  {
+    "ct": 1312761600,
+    "c": "7.9"
+  },
+  {
+    "ct": 1313366400,
+    "c": "10.7957"
+  },
+  {
+    "ct": 1313971200,
+    "c": "11.31125"
+  },
+  {
+    "ct": 1314576000,
+    "c": "9.07011"
+  },
+  {
+    "ct": 1315180800,
+    "c": "8.17798"
+  },
+  {
+    "ct": 1315785600,
+    "c": "5.86436"
+  },
+  {
+    "ct": 1316390400,
+    "c": "5.2"
+  },
+  {
+    "ct": 1316995200,
+    "c": "5.33"
+  },
+  {
+    "ct": 1317600000,
+    "c": "5.02701"
+  },
+  {
+    "ct": 1318204800,
+    "c": "4.10288"
+  },
+  {
+    "ct": 1318809600,
+    "c": "3.5574"
+  },
+  {
+    "ct": 1319414400,
+    "c": "3.12657"
+  },
+  {
+    "ct": 1320019200,
+    "c": "3.27"
+  },
+  {
+    "ct": 1320624000,
+    "c": "2.95959"
+  },
+  {
+    "ct": 1321228800,
+    "c": "2.99626"
+  },
+  {
+    "ct": 1321833600,
+    "c": "2.2"
+  },
+  {
+    "ct": 1322438400,
+    "c": "2.47991"
+  },
+  {
+    "ct": 1323043200,
+    "c": "2.82809"
+  },
+  {
+    "ct": 1323648000,
+    "c": "3.2511"
+  },
+  {
+    "ct": 1324252800,
+    "c": "3.193"
+  },
+  {
+    "ct": 1324857600,
+    "c": "4.225"
+  },
+  {
+    "ct": 1325462400,
+    "c": "5.26766"
+  },
+  {
+    "ct": 1326067200,
+    "c": "7.11358"
+  },
+  {
+    "ct": 1326672000,
+    "c": "7.00177"
+  },
+  {
+    "ct": 1327276800,
+    "c": "6.3097"
+  },
+  {
+    "ct": 1327881600,
+    "c": "5.38191"
+  },
+  {
+    "ct": 1328486400,
+    "c": "5.68881"
+  },
+  {
+    "ct": 1329091200,
+    "c": "5.51468"
+  },
+  {
+    "ct": 1329696000,
+    "c": "4.38669"
+  },
+  {
+    "ct": 1330300800,
+    "c": "4.922"
+  },
+  {
+    "ct": 1330905600,
+    "c": "4.8201"
+  },
+  {
+    "ct": 1331510400,
+    "c": "4.90901"
+  },
+  {
+    "ct": 1332115200,
+    "c": "5.27943"
+  },
+  {
+    "ct": 1332720000,
+    "c": "4.55001"
+  },
+  {
+    "ct": 1333324800,
+    "c": "4.81922"
+  },
+  {
+    "ct": 1333929600,
+    "c": "4.79253"
+  },
+  {
+    "ct": 1334534400,
+    "c": "4.96892"
+  },
+  {
+    "ct": 1335139200,
+    "c": "5.20352"
+  },
+  {
+    "ct": 1335744000,
+    "c": "4.90441"
+  },
+  {
+    "ct": 1336348800,
+    "c": "5.04991"
+  },
+  {
+    "ct": 1336953600,
+    "c": "4.92996"
+  },
+  {
+    "ct": 1337558400,
+    "c": "5.09002"
+  },
+  {
+    "ct": 1338163200,
+    "c": "5.13896"
+  },
+  {
+    "ct": 1338768000,
+    "c": "5.2051"
+  },
+  {
+    "ct": 1339372800,
+    "c": "5.46829"
+  },
+  {
+    "ct": 1339977600,
+    "c": "6.16382"
+  },
+  {
+    "ct": 1340582400,
+    "c": "6.35002"
+  },
+  {
+    "ct": 1341187200,
+    "c": "6.62898"
+  },
+  {
+    "ct": 1341792000,
+    "c": "6.79898"
+  },
+  {
+    "ct": 1342396800,
+    "c": "7.62101"
+  },
+  {
+    "ct": 1343001600,
+    "c": "8.4096"
+  },
+  {
+    "ct": 1343606400,
+    "c": "8.71027"
+  },
+  {
+    "ct": 1344211200,
+    "c": "10.86998"
+  },
+  {
+    "ct": 1344816000,
+    "c": "11.6239"
+  },
+  {
+    "ct": 1345420800,
+    "c": "7.98"
+  },
+  {
+    "ct": 1346025600,
+    "c": "10.61"
+  },
+  {
+    "ct": 1346630400,
+    "c": "10.2041"
+  },
+  {
+    "ct": 1347235200,
+    "c": "11.02"
+  },
+  {
+    "ct": 1347840000,
+    "c": "11.87"
+  },
+  {
+    "ct": 1348444800,
+    "c": "12.19331"
+  },
+  {
+    "ct": 1349049600,
+    "c": "12.4"
+  },
+  {
+    "ct": 1349654400,
+    "c": "11.8034"
+  },
+  {
+    "ct": 1350259200,
+    "c": "11.7389"
+  },
+  {
+    "ct": 1350864000,
+    "c": "11.63107"
+  },
+  {
+    "ct": 1351468800,
+    "c": "10.69998"
+  },
+  {
+    "ct": 1352073600,
+    "c": "10.80011"
+  },
+  {
+    "ct": 1352678400,
+    "c": "10.84692"
+  },
+  {
+    "ct": 1353283200,
+    "c": "11.65961"
+  },
+  {
+    "ct": 1353888000,
+    "c": "12.4821"
+  },
+  {
+    "ct": 1354492800,
+    "c": "12.50003"
+  },
+  {
+    "ct": 1355097600,
+    "c": "13.388"
+  },
+  {
+    "ct": 1355702400,
+    "c": "13.30002"
+  },
+  {
+    "ct": 1356307200,
+    "c": "13.31202"
+  },
+  {
+    "ct": 1356912000,
+    "c": "13.45001"
+  },
+  {
+    "ct": 1357516800,
+    "c": "13.5199"
+  },
+  {
+    "ct": 1358121600,
+    "c": "14.11601"
+  },
+  {
+    "ct": 1358726400,
+    "c": "15.7"
+  },
+  {
+    "ct": 1359331200,
+    "c": "17.95"
+  },
+  {
+    "ct": 1359936000,
+    "c": "20.59"
+  },
+  {
+    "ct": 1360540800,
+    "c": "23.96975"
+  },
+  {
+    "ct": 1361145600,
+    "c": "26.8146"
+  },
+  {
+    "ct": 1361750400,
+    "c": "29.88999"
+  },
+  {
+    "ct": 1362355200,
+    "c": "34.49999"
+  },
+  {
+    "ct": 1362960000,
+    "c": "46"
+  },
+  {
+    "ct": 1363564800,
+    "c": "47.4"
+  },
+  {
+    "ct": 1364169600,
+    "c": "71.93"
+  },
+  {
+    "ct": 1364774400,
+    "c": "93.03001"
+  },
+  {
+    "ct": 1365379200,
+    "c": "162.30102"
+  },
+  {
+    "ct": 1365984000,
+    "c": "89.99999"
+  },
+  {
+    "ct": 1366588800,
+    "c": "119.2"
+  },
+  {
+    "ct": 1367193600,
+    "c": "134.44444"
+  },
+  {
+    "ct": 1367798400,
+    "c": "115.98"
+  },
+  {
+    "ct": 1368403200,
+    "c": "114.82002"
+  },
+  {
+    "ct": 1369008000,
+    "c": "122.49999"
+  },
+  {
+    "ct": 1369612800,
+    "c": "133.5"
+  },
+  {
+    "ct": 1370217600,
+    "c": "122.5"
+  },
+  {
+    "ct": 1370822400,
+    "c": "100.43743"
+  },
+  {
+    "ct": 1371427200,
+    "c": "99.9"
+  },
+  {
+    "ct": 1372032000,
+    "c": "107.90001"
+  },
+  {
+    "ct": 1372636800,
+    "c": "97.51"
+  },
+  {
+    "ct": 1373241600,
+    "c": "76.5"
+  },
+  {
+    "ct": 1373846400,
+    "c": "94.41986"
+  },
+  {
+    "ct": 1374451200,
+    "c": "91.998"
+  },
+  {
+    "ct": 1375056000,
+    "c": "98.78008"
+  },
+  {
+    "ct": 1375660800,
+    "c": "105.12"
+  },
+  {
+    "ct": 1376265600,
+    "c": "105"
+  },
+  {
+    "ct": 1376870400,
+    "c": "113.38"
+  },
+  {
+    "ct": 1377475200,
+    "c": "122.11102"
+  },
+  {
+    "ct": 1378080000,
+    "c": "146.01003"
+  },
+  {
+    "ct": 1378684800,
+    "c": "126.31501"
+  },
+  {
+    "ct": 1379289600,
+    "c": "138.3002"
+  },
+  {
+    "ct": 1379894400,
+    "c": "134.00001"
+  },
+  {
+    "ct": 1380499200,
+    "c": "143.88402"
+  },
+  {
+    "ct": 1381104000,
+    "c": "137.8"
+  },
+  {
+    "ct": 1381708800,
+    "c": "147.53"
+  },
+  {
+    "ct": 1382313600,
+    "c": "186.1"
+  },
+  {
+    "ct": 1382918400,
+    "c": "207.0001"
+  },
+  {
+    "ct": 1383523200,
+    "c": "224.01001"
+  },
+  {
+    "ct": 1384128000,
+    "c": "336.33101"
+  },
+  {
+    "ct": 1384732800,
+    "c": "528"
+  },
+  {
+    "ct": 1385337600,
+    "c": "795"
+  },
+  {
+    "ct": 1385942400,
+    "c": "1004.42392"
+  },
+  {
+    "ct": 1386547200,
+    "c": "804.5"
+  },
+  {
+    "ct": 1387152000,
+    "c": "919.985"
+  },
+  {
+    "ct": 1387756800,
+    "c": "639.48"
+  },
+  {
+    "ct": 1388361600,
+    "c": "786.98"
+  },
+  {
+    "ct": 1388966400,
+    "c": "1015"
+  },
+  {
+    "ct": 1389571200,
+    "c": "940"
+  },
+  {
+    "ct": 1390176000,
+    "c": "954.995"
+  },
+  {
+    "ct": 1390780800,
+    "c": "1007.98999"
+  },
+  {
+    "ct": 1391385600,
+    "c": "954"
+  },
+  {
+    "ct": 1391990400,
+    "c": "659.49776"
+  },
+  {
+    "ct": 1392595200,
+    "c": "299.702"
+  },
+  {
+    "ct": 1393200000,
+    "c": "310.00001"
+  },
+  {
+    "ct": 1393804800,
+    "c": "135"
+  }
+]

backend/src/tasks/price-updater.ts

@@ -0,0 +1,262 @@
+import * as fs from 'fs';
+import config from '../config';
+import logger from '../logger';
+import PricesRepository from '../repositories/PricesRepository';
+import BitfinexApi from './price-feeds/bitfinex-api';
+import BitflyerApi from './price-feeds/bitflyer-api';
+import CoinbaseApi from './price-feeds/coinbase-api';
+import FtxApi from './price-feeds/ftx-api';
+import GeminiApi from './price-feeds/gemini-api';
+import KrakenApi from './price-feeds/kraken-api';
+
+export interface PriceFeed {
+  name: string;
+  url: string;
+  urlHist: string;
+  currencies: string[];
+
+  $fetchPrice(currency): Promise<number>;
+  $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory>;
+}
+
+export interface PriceHistory {
+  [timestamp: number]: Prices;
+}
+
+export interface Prices {
+  USD: number;
+  EUR: number;
+  GBP: number;
+  CAD: number;
+  CHF: number;
+  AUD: number;
+  JPY: number;
+}
+
+class PriceUpdater {
+  historyInserted: boolean = false;
+  lastRun: number = 0;
+  lastHistoricalRun: number = 0;
+  running: boolean = false;
+  feeds: PriceFeed[] = [];
+  currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+  latestPrices: Prices;
+
+  constructor() {
+    this.latestPrices = this.getEmptyPricesObj();
+
+    this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
+    this.feeds.push(new FtxApi());
+    this.feeds.push(new KrakenApi());
+    this.feeds.push(new CoinbaseApi());
+    this.feeds.push(new BitfinexApi());
+    this.feeds.push(new GeminiApi());
+  }
+
+  public getEmptyPricesObj(): Prices {
+    return {
+      USD: -1,
+      EUR: -1,
+      GBP: -1,
+      CAD: -1,
+      CHF: -1,
+      AUD: -1,
+      JPY: -1,
+    };
+  }
+
+  public async $run(): Promise<void> {
+    if (this.running === true) {
+      return;
+    }
+    this.running = true;
+
+    if ((Math.round(new Date().getTime() / 1000) - this.lastHistoricalRun) > 3600 * 24) {
+      // Once a day, look for missing prices (could happen due to network connectivity issues)
+      this.historyInserted = false;
+    }
+
+    try {
+      if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
+        await this.$insertHistoricalPrices();
+      } else {
+        await this.$updatePrice();
+      }
+    } catch (e) {
+      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+
+    this.running = false;
+  }
+
+  /**
+   * Fetch last BTC price from exchanges, average them, and save it in the database once every hour
+   */
+  private async $updatePrice(): Promise<void> {
+    if (this.lastRun === 0 && config.DATABASE.ENABLED === true) {
+      this.lastRun = await PricesRepository.$getLatestPriceTime();
+    }
+
+    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) {
+      // Refresh only once every hour
+      return;
+    }
+
+    const previousRun = this.lastRun;
+    this.lastRun = new Date().getTime() / 1000;
+
+    for (const currency of this.currencies) {
+      let prices: number[] = [];
+
+      for (const feed of this.feeds) {
+        // Fetch prices from API which supports `currency`
+        if (feed.currencies.includes(currency)) {
+          try {
+            const price = await feed.$fetchPrice(currency);
+            if (price > 0) {
+              prices.push(price);
+            }
+            logger.debug(`${feed.name} BTC/${currency} price: ${price}`);
+          } catch (e) {
+            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`);
+          }
+        }
+      }
+      if (prices.length === 1) {
+        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`);
+      }
+
+      // Compute average price, non weighted
+      prices = prices.filter(price => price > 0);
+      this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+    }
+
+    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
+
+    if (config.DATABASE.ENABLED === true) {
+      // Save everything in db
+      try {
+        const p = 60 * 60 * 1000; // milliseconds in an hour
+        const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
+        await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
+      } catch (e) {
+        this.lastRun = previousRun + 5 * 60;
+        logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
+      }
+    }
+
+    this.lastRun = new Date().getTime() / 1000;
+  }
+
+  /**
+   * Called once by the database migration to initialize historical prices data (weekly)
+   * We use MtGox weekly price from July 19, 2010 to September 30, 2013
+   * We use Kraken weekly price from October 3, 2013 up to last month
+   * We use Kraken hourly price for the past month
+   */
+  private async $insertHistoricalPrices(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    // Insert MtGox weekly prices
+    const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
+    const prices = this.getEmptyPricesObj();
+    let insertedCount: number = 0;
+    for (const price of pricesJson) {
+      if (existingPriceTimes.includes(price['ct'])) {
+        continue;
+      }
+
+      // From 1380758400 we will use Kraken price as it follows closely MtGox, but was not affected as much
+      // by the MtGox exchange collapse a few months later
+      if (price['ct'] > 1380758400) {
+        break;
+      }
+      prices.USD = price['c'];
+      await PricesRepository.$savePrices(price['ct'], prices);
+      ++insertedCount;
+    }
+    if (insertedCount > 0) {
+      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+    } else {
+      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+    }
+
+    // Insert Kraken weekly prices
+    await new KrakenApi().$insertHistoricalPrice();
+
+    // Insert missing recent hourly prices
+    await this.$insertMissingRecentPrices();
+
+    this.historyInserted = true;
+    this.lastHistoricalRun = new Date().getTime();
+  }
+
+  /**
+   * Find missing hourly prices and insert them in the database
+   * It has a limited backward range and it depends on which API are available
+   */
+  private async $insertMissingRecentPrices(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    logger.info(`Fetching hourly price history from exchanges and saving missing ones into the database, this may take a while`);
+
+    const historicalPrices: PriceHistory[] = [];
+
+    // Fetch all historical hourly prices
+    for (const feed of this.feeds) {
+      try {
+        historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies));
+      } catch (e) {
+        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
+      }
+    }
+
+    // Group them by timestamp and currency, for example
+    // grouped[123456789]['USD'] = [1, 2, 3, 4];
+    const grouped: Object = {};
+    for (const historicalEntry of historicalPrices) {
+      for (const time in historicalEntry) {
+        if (existingPriceTimes.includes(parseInt(time, 10))) {
+          continue;
+        }
+
+        if (grouped[time] === undefined) {
+          grouped[time] = {
+            USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: []
+          };
+        }
+
+        for (const currency of this.currencies) {
+          const price = historicalEntry[time][currency];
+          if (price > 0) {
+            grouped[time][currency].push(parseInt(price, 10));
+          }
+        }
+      }
+    }
+
+    // Average prices and insert everything into the db
+    let totalInserted = 0;
+    for (const time in grouped) {
+      const prices: Prices = this.getEmptyPricesObj();
+      for (const currency in grouped[time]) {
+        if (grouped[time][currency].length === 0) {
+          continue;
+        }
+        prices[currency] = Math.round((grouped[time][currency].reduce(
+          (partialSum, a) => partialSum + a, 0)
+        ) / grouped[time][currency].length);
+      }
+      await PricesRepository.$savePrices(parseInt(time, 10), prices);
+      ++totalInserted;
+    }
+
+    if (totalInserted > 0) {
+      logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`);
+    } else {
+      logger.debug(`Inserted ${totalInserted} hourly historical prices into the db`);
+    }
+  }
+}
+
+export default new PriceUpdater();

backend/src/utils/axios-query.ts

@@ -0,0 +1,63 @@
+import axios, { AxiosResponse } from 'axios';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import backendInfo from '../api/backend-info';
+import config from '../config';
+import logger from '../logger';
+import * as https from 'https';
+
+export async function query(path): Promise<object | undefined> {
+ type axiosOptions = {
+   headers: {
+     'User-Agent': string
+   };
+   timeout: number;
+   httpsAgent?: https.Agent;
+ };
+ const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+ const axiosOptions: axiosOptions = {
+   headers: {
+     'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+   },
+   timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+ };
+ let retry = 0;
+
+ while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+   try {
+     if (config.SOCKS5PROXY.ENABLED) {
+       const socksOptions: any = {
+         agentOptions: {
+           keepAlive: true,
+         },
+         hostname: config.SOCKS5PROXY.HOST,
+         port: config.SOCKS5PROXY.PORT
+       };
+
+       if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+         socksOptions.username = config.SOCKS5PROXY.USERNAME;
+         socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+       } else {
+         // Retry with different tor circuits https://stackoverflow.com/a/64960234
+         socksOptions.username = `circuit${retry}`;
+       }
+
+       axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+     }
+
+     const data: AxiosResponse = await axios.get(path, axiosOptions);
+     if (data.statusText === 'error' || !data.data) {
+       throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
+     }
+     return data.data;
+   } catch (e) {
+     logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
+     retry++;
+   }
+   if (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+     await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+   }
+ }
+
+ logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
+ return undefined;
+}

backend/src/utils/blocks-utils.ts

@@ -0,0 +1,32 @@
+import { BlockExtended } from '../mempool.interfaces';
+
+export function prepareBlock(block: any): BlockExtended {
+  return <BlockExtended>{
+    id: block.id ?? block.hash, // hash for indexed block
+    timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
+    height: block.height,
+    version: block.version,
+    bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
+    nonce: block.nonce,
+    difficulty: block.difficulty,
+    merkle_root: block.merkle_root ?? block.merkleroot,
+    tx_count: block.tx_count ?? block.nTx,
+    size: block.size,
+    weight: block.weight,
+    previousblockhash: block.previousblockhash,
+    extras: {
+      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
+      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
+      feeRange: block.feeRange ?? block.fee_span,
+      reward: block.reward ?? block?.extras?.reward,
+      totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
+      avgFee: block?.extras?.avgFee ?? block.avg_fee,
+      avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
+      pool: block?.extras?.pool ?? (block?.pool_id ? {
+        id: block.pool_id,
+        name: block.pool_name,
+        slug: block.pool_slug,
+      } : undefined),
+    }
+  };
+}

backend/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "types": ["node"],
     "module": "commonjs",
     "target": "esnext",
     "lib": ["es2019", "dom"],
@@ -10,7 +11,9 @@
     "moduleResolution": "node",
     "typeRoots": [
       "node_modules/@types"
-    ]
+    ],
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true
   },
   "include": [
     "src/**/*.ts"

backend/tslint.json

@@ -1,137 +0,0 @@
-{
-  "rules": {
-    "arrow-return-shorthand": true,
-    "callable-types": true,
-    "class-name": true,
-    "comment-format": [
-      true,
-      "check-space"
-    ],
-    "curly": true,
-    "deprecation": {
-      "severity": "warn"
-    },
-    "eofline": true,
-    "forin": false,
-    "import-blacklist": [
-      true,
-      "rxjs",
-      "rxjs/Rx"
-    ],
-    "import-spacing": true,
-    "indent": [
-      true,
-      "spaces"
-    ],
-    "interface-over-type-literal": true,
-    "label-position": true,
-    "max-line-length": [
-      true,
-      140
-    ],
-    "member-access": false,
-    "member-ordering": [
-      true,
-      {
-        "order": [
-          "static-field",
-          "instance-field",
-          "static-method",
-          "instance-method"
-        ]
-      }
-    ],
-    "no-arg": true,
-    "no-bitwise": true,
-    "no-console": [
-      true,
-      "debug",
-      "info",
-      "time",
-      "timeEnd",
-      "trace"
-    ],
-    "no-construct": true,
-    "no-debugger": true,
-    "no-duplicate-super": true,
-    "no-empty": false,
-    "no-empty-interface": true,
-    "no-eval": true,
-    "no-inferrable-types": false,
-    "no-misused-new": true,
-    "no-non-null-assertion": true,
-    "no-shadowed-variable": true,
-    "no-string-literal": false,
-    "no-string-throw": true,
-    "no-switch-case-fall-through": true,
-    "no-trailing-whitespace": true,
-    "no-unnecessary-initializer": true,
-    "no-unused-expression": true,
-    "no-use-before-declare": true,
-    "no-var-keyword": true,
-    "object-literal-sort-keys": false,
-    "one-line": [
-      true,
-      "check-open-brace",
-      "check-catch",
-      "check-else",
-      "check-whitespace"
-    ],
-    "prefer-const": true,
-    "quotemark": [
-      true,
-      "single"
-    ],
-    "radix": true,
-    "semicolon": [
-      true,
-      "always"
-    ],
-    "triple-equals": [
-      true,
-      "allow-null-check"
-    ],
-    "typedef-whitespace": [
-      true,
-      {
-        "call-signature": "nospace",
-        "index-signature": "nospace",
-        "parameter": "nospace",
-        "property-declaration": "nospace",
-        "variable-declaration": "nospace"
-      }
-    ],
-    "unified-signatures": true,
-    "variable-name": false,
-    "whitespace": [
-      true,
-      "check-branch",
-      "check-decl",
-      "check-operator",
-      "check-separator",
-      "check-type"
-    ],
-    "directive-selector": [
-      true,
-      "attribute",
-      "app",
-      "camelCase"
-    ],
-    "component-selector": [
-      true,
-      "element",
-      "app",
-      "kebab-case"
-    ],
-    "no-output-on-prefix": true,
-    "use-input-property-decorator": true,
-    "use-output-property-decorator": true,
-    "use-host-property-decorator": true,
-    "no-input-rename": true,
-    "no-output-rename": true,
-    "use-life-cycle-interface": true,
-    "use-pipe-transform-interface": true,
-    "component-class-suffix": true,
-    "directive-class-suffix": true
-  }
-}

contributors/TechMiX.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: TechMiX

contributors/antonilol.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: antonilol

contributors/ayanamidev.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 15, 2022.
+
+Signed: ayanamidev

contributors/bosch-0.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: Bosch-0

contributors/dsbaars.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: dsbaars

contributors/emzy.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @emzy as of January 25, 2022

contributors/erikarvstedt.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 6, 2022.
+
+Signed: erikarvstedt

contributors/hunicus.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @hunicus as of January 25, 2022

contributors/knorrium.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @knorrium as of January 25, 2022

contributors/miguelmedeiros.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @miguelmedeiros as of January 25, 2022

contributors/mononaut.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 31, 2022.
+
+Signed: mononaut

contributors/naveensrinivasan.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of March 11, 2022.
+
+Signed: naveensrinivasan

contributors/nymkappa.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @nymkappa as of January 25, 2022