Merge pull request #2801 from knorrium/use_fra_for_bisq

Handle ISP with no nodes

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: ['https://mempool.space/sponsor'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/dependabot.yml

@@ -1,20 +1,28 @@
 version: 2
 updates:
-- package-ecosystem: npm
-  directory: "/backend"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-- package-ecosystem: npm
-  directory: "/frontend"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-- package-ecosystem: docker
-  directory: "/docker/backend"
-  schedule:
-    interval: weekly
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-      interval: "weekly"
+  - package-ecosystem: npm
+    directory: "/backend"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - update-types: ["version-update:semver-major"]
+  - package-ecosystem: npm
+    directory: "/frontend"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - update-types: ["version-update:semver-major"]
+  - package-ecosystem: docker
+    directory: "/docker/backend"
+    schedule:
+      interval: daily
+    ignore:
+      - update-types: ["version-update:semver-major"]
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: daily
+    ignore:
+      - update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -0,0 +1,96 @@
+name: CI Pipeline for the Backend and Frontend
+
+on:
+  pull_request:
+    types: [opened, review_requested, synchronize]
+
+jobs:
+  backend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.16.0", "18.5.0"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Install (Prod dependencies only)
+        if: ${{ matrix.flavor == 'prod'}}
+        run: npm ci --omit=dev --omit=optional
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Unit Tests
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run test
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+  frontend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.15.0", "18.5.0"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install (Prod dependencies only)
+        run: npm ci --omit=dev --omit=optional
+        if: ${{ matrix.flavor == 'prod'}}
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      # - name: Test
+      #   run: npm run test
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend

.github/workflows/cypress.yml

@@ -1,88 +1,58 @@
 name: Cypress Tests
 
-on: [push, pull_request]
+on:
+  pull_request:
+    types: [ opened, review_requested, synchronize ]
 jobs:
   cypress:
-    runs-on: ${{ matrix.os }}
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    runs-on: "ubuntu-latest"
     strategy:
       fail-fast: false
       matrix:
-        containers: [1, 2, 3, 4, 5]
-        os: ["ubuntu-latest"]
-        browser: [chrome]
-    name: E2E tests on ${{ matrix.browser }} - ${{ matrix.os }}
+        module: ["mempool", "liquid", "bisq"]
+        include:
+          - module: "mempool"
+            spec: |
+              cypress/e2e/mainnet/*.spec.ts
+              cypress/e2e/signet/*.spec.ts
+              cypress/e2e/testnet/*.spec.ts
+          - module: "liquid"
+            spec: |
+              cypress/e2e/liquid/liquid.spec.ts
+              cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+          - module: "bisq"
+            spec: |
+              cypress/e2e/bisq/bisq.spec.ts
+          
+    name: E2E tests for ${{ matrix.module }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          path: ${{ matrix.module }}
+          
       - name: Setup node
         uses: actions/setup-node@v2
         with:
           node-version: 16.15.0
           cache: 'npm'
-          cache-dependency-path: frontend/package-lock.json
-      - name: ${{ matrix.browser }} browser tests (Mempool)
-        uses: cypress-io/github-action@v4
-        with:
-          tag: ${{ github.event_name }}
-          working-directory: frontend
-          build: npm run config:defaults:mempool
-          start: npm run start:local-staging
-          wait-on: 'http://localhost:4200'
-          wait-on-timeout: 120
-          record: true
-          parallel: true
-          spec: |
-            cypress/e2e/mainnet/*.spec.ts
-            cypress/e2e/signet/*.spec.ts
-            cypress/e2e/testnet/*.spec.ts
-          group: Tests on ${{ matrix.browser }} (Mempool)
-          browser: ${{ matrix.browser }}
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
-        env:
-          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+          cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 
-      - name: ${{ matrix.browser }} browser tests (Liquid)
+      - name: Chrome browser tests (${{ matrix.module }})
         uses: cypress-io/github-action@v4
-        if: always()
         with:
           tag: ${{ github.event_name }}
-          working-directory: frontend
-          build: npm run config:defaults:liquid
+          working-directory: ${{ matrix.module }}/frontend
+          build: npm run config:defaults:${{ matrix.module }}
           start: npm run start:local-staging
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
-          spec: |
-            cypress/e2e/liquid/liquid.spec.ts
-            cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
-          group: Tests on ${{ matrix.browser }} (Liquid)
-          browser: ${{ matrix.browser }}
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
-        env:
-          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
-
-      - name: ${{ matrix.browser }} browser tests (Bisq)
-        uses: cypress-io/github-action@v4
-        if: always()
-        with:
-          tag: ${{ github.event_name }}
-          working-directory: frontend
-          build: npm run config:defaults:bisq
-          start: npm run start:local-staging
-          wait-on: 'http://localhost:4200'
-          wait-on-timeout: 120
-          record: true
-          parallel: true
-          spec: cypress/e2e/bisq/bisq.spec.ts
-          group: Tests on ${{ matrix.browser }} (Bisq)
-          browser: ${{ matrix.browser }}
+          spec: ${{ matrix.spec }}
+          group: Tests on Chrome (${{ matrix.module }})
+          browser: "chrome"
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
           COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}

.github/workflows/on-tag.yml

@@ -1,7 +1,7 @@
 name: Docker build on tag
 env:
   DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
   DOCKER_BUILDKIT: 0
   COMPOSE_DOCKER_CLI_BUILD: 0
 
@@ -21,16 +21,46 @@ jobs:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
     name: Build and push to DockerHub
     steps:
+      # Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
+      - name: Replace the current swap file
+        shell: bash
+        run: |
+          sudo swapoff /mnt/swapfile
+          sudo rm -v /mnt/swapfile
+          sudo fallocate -l 10G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+
+      - name: Show current memory and swap status
+        shell: bash
+        run: |
+          sudo free -h
+          echo
+          sudo swapon --show
+
+      - name: Mount a tmpfs over /var/lib/docker
+        shell: bash
+        run: |
+          if [ ! -d "/var/lib/docker" ]; then
+            echo "Directory '/var/lib/docker' not found"
+            exit 1
+          fi
+          sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
+          sudo systemctl restart docker
+          sudo df -h | grep docker
+
       - name: Set env variables
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Show set environment variables
         run: |
           printf "      TAG: %s\n"  "$TAG"
-      
+
       - name: Add SHORT_SHA env property with commit short sha
         run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
 
@@ -38,24 +68,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v2.2.1
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@661fd3eb7f2f20d8c7c84bc2b0509efd7a826628 # v2
+        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11
         id: cache
         with:
           path: /tmp/.buildx-cache

.gitignore

@@ -2,3 +2,6 @@ sitemap
 data
 docker-compose.yml
 backend/mempool-config.json
+*.swp
+frontend/src/resources/config.template.js
+frontend/src/resources/config.js

.nvmrc

@@ -1 +1 @@
-v16.15.0
+v16.16.0

CONTRIBUTING.md

@@ -6,6 +6,8 @@ In order to clarify the intellectual property license granted with Contributions
 
 When submitting a pull request for the first time, please create a file with a name like `/contributors/{github_username}.txt`, and in the content of that file indicate your agreement to the Contributor License Agreement terms below. An example of what that file should contain can be seen in wiz's agreement file. (This method of CLA "signing" is borrowed from Medium's open source project.)
 
+Also, please GPG-sign all your commits (`git config commit.gpgsign true`).
+
 # Contributor License Agreement
 
 Last Updated: January 25, 2022

README.md

@@ -29,5 +29,5 @@ Mempool can be conveniently installed on the following full-node distros:
 Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
 - See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
-- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers and small-scale deployments.
-- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.
+- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers.
+- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.

backend/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
+

backend/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

backend/.eslintrc

@@ -0,0 +1,37 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 0,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "@typescript-eslint/explicit-function-return-type": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1,
+    "quotes": [1, "single", { "allowTemplateLiterals": true }],
+    "semi": 1,
+    "eqeqeq": 1
+  }
+}

backend/.gitignore

@@ -1,9 +1,10 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # production config and external assets
-*.json
+!mempool-config.template.json
 !mempool-config.sample.json
-
+mempool-config.json
+pools.json
 icons.json
 
 # compiled output

backend/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

backend/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "endOfLine": "lf",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

backend/README.md

@@ -1,6 +1,6 @@
 # Mempool Backend
 
-These instructions are mostly intended for developers, but can be used as a basis for personal or small-scale production setups. 
+These instructions are mostly intended for developers. 
 
 If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
 
@@ -77,13 +77,13 @@ Query OK, 0 rows affected (0.00 sec)
 
 #### Build
 
-_Make sure to use Node.js 16.15 and npm 7._
+_Make sure to use Node.js 16.10 and npm 7._
 
 Install dependencies with `npm` and build the backend:
 
 ```
 cd backend
-npm install # add --prod for production
+npm install
 npm run build
 ```
 
@@ -110,6 +110,11 @@ Run the Mempool backend:
 
 ```
 npm run start
+
+```
+You can also set env var `MEMPOOL_CONFIG_FILE` to specify a custom config file location:
+```
+MEMPOOL_CONFIG_FILE=/path/to/mempool-config.json npm run start
 ```
 
 When it's running, you should see output like this:
@@ -159,3 +164,57 @@ nodemon src/index.ts --ignore cache/ --ignore pools.json
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
+
+### Useful Regtest Commands
+
+Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe223a
+
+Run bitcoind on regtest:
+   ```
+   bitcoind -regtest -rpcport=8332
+   ```
+
+Create a new wallet, if needed:
+   ```
+   bitcoin-cli -regtest -rpcport=8332 createwallet test
+   ```
+
+Load wallet (this command may take a while if you have lot of UTXOs):
+   ```
+   bitcoin-cli -regtest -rpcport=8332 loadwallet test
+   ```
+
+Get a new address:
+   ```
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   ```
+
+Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
+   ```
+   bitcoin-cli -regtest -rpcport=8332 generatetoaddress 101 $address
+   ```
+
+Send 0.1 BTC at 5 sat/vB to another address:
+   ```
+   ./src/bitcoin-cli -named -regtest -rpcport=8332 sendtoaddress address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress) amount=0.1 fee_rate=5
+   ```
+
+See more example of `sendtoaddress`:
+   ```
+   ./src/bitcoin-cli sendtoaddress # will print the help
+   ```
+
+Mini script to generate transactions with random TX fee-rate (between 1 to 100 sat/vB). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
+   ```
+   #!/bin/bash
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   for i in {1..1000000}
+   do
+     ./src/bitcoin-cli -regtest -rpcport=8332 -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1  1 100)
+   done
+   ```
+
+Generate block at regular interval (every 10 seconds in this example):
+   ```
+   watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
+   ```

backend/jest.config.ts

@@ -0,0 +1,20 @@
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  verbose: true,
+  automock: false,
+  collectCoverage: true,
+  collectCoverageFrom: ["./src/**/**.ts"],
+  coverageProvider: "babel",
+  coverageThreshold: {
+    global: {
+      lines: 1
+    }
+  },
+  setupFiles: [
+    "./testSetup.ts",
+  ],
+}
+export default config;

backend/mempool-config.sample.json

@@ -2,6 +2,7 @@
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
@@ -13,13 +14,20 @@
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "INDEXING_BLOCKS_AMOUNT": 11000,
+    "BLOCKS_SUMMARIES_INDEXING": false,
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": [],
     "EXTERNAL_MAX_RETRY": 1,
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
-    "STDOUT_LOG_MIN_PRIORITY": "debug"
+    "STDOUT_LOG_MIN_PRIORITY": "debug",
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "ADVANCED_GBT_AUDIT": false,
+    "ADVANCED_GBT_MEMPOOL": false,
+    "TRANSACTION_INDEXING": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -61,10 +69,33 @@
     "ENABLED": true,
     "TX_PER_SECOND_SAMPLE_PERIOD": 150
   },
+  "MAXMIND": {
+    "ENABLED": false,
+    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
+  },
   "BISQ": {
     "ENABLED": false,
     "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
   },
+  "LIGHTNING": {
+    "ENABLED": false,
+    "BACKEND": "lnd",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": 20
+  },
+  "LND": {
+    "TLS_CERT_PATH": "tls.cert",
+    "MACAROON_PATH": "readonly.macaroon",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "lightning-rpc"
+  },
   "SOCKS5PROXY": {
     "ENABLED": false,
     "USE_ONION": true,

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.4.0-dev",
+  "version": "2.5.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -16,33 +16,53 @@
     "mempool",
     "blockchain",
     "explorer",
-    "liquid"
+    "liquid",
+    "lightning"
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
-    "tsc": "./node_modules/typescript/bin/tsc",
-    "build": "npm run tsc",
+    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
+    "build": "npm run tsc && npm run create-resources",
+    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
+    "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "./node_modules/.bin/jest --coverage",
+    "lint": "./node_modules/.bin/eslint . --ext .ts",
+    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
+    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
+    "@babel/core": "^7.20.5",
     "@mempool/electrum-client": "^1.1.7",
+    "@types/node": "^16.11.41",
     "axios": "~0.27.2",
-    "bitcoinjs-lib": "6.0.1",
-    "crypto-js": "^4.0.0",
-    "express": "^4.18.0",
-    "mysql2": "2.3.3",
-    "node-worker-threads-pool": "^1.5.1",
-    "socks-proxy-agent": "^6.2.0",
-    "typescript": "~4.7.2",
-    "ws": "~8.7.0"
+    "bitcoinjs-lib": "~6.0.2",
+    "crypto-js": "~4.1.1",
+    "express": "~4.18.2",
+    "maxmind": "~4.3.8",
+    "mysql2": "~2.3.3",
+    "node-worker-threads-pool": "~1.5.1",
+    "socks-proxy-agent": "~7.0.0",
+    "typescript": "~4.7.4",
+    "ws": "~8.11.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.20.5",
+    "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
+    "@types/crypto-js": "^4.1.1",
+    "@types/express": "^4.17.14",
+    "@types/jest": "^29.2.3",
     "@types/ws": "~8.5.3",
-    "@types/express": "^4.17.13",
-    "tslint": "^6.1.0"
+    "@typescript-eslint/eslint-plugin": "^5.45.0",
+    "@typescript-eslint/parser": "^5.45.0",
+    "eslint": "^8.28.0",
+    "eslint-config-prettier": "^8.5.0",
+    "jest": "^29.3.1",
+    "prettier": "^2.8.0",
+    "ts-jest": "^29.0.3",
+    "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -0,0 +1,115 @@
+{
+  "MEMPOOL": {
+    "ENABLED": true,
+    "NETWORK": "__MEMPOOL_NETWORK__",
+    "BACKEND": "__MEMPOOL_BACKEND__",
+    "ENABLED": true,
+    "BLOCKS_SUMMARIES_INDEXING": true,
+    "HTTP_PORT": 1,
+    "SPAWN_CLUSTER_PROCS": 2,
+    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
+    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "POLL_RATE_MS": 3,
+    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CLEAR_PROTECTION_MINUTES": 4,
+    "RECOMMENDED_FEE_PERCENTILE": 5,
+    "BLOCK_WEIGHT_UNITS": 6,
+    "INITIAL_BLOCKS_AMOUNT": 7,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "PRICE_FEED_UPDATE_INTERVAL": 9,
+    "USE_SECOND_NODE_FOR_MINFEE": 10,
+    "EXTERNAL_ASSETS": 11,
+    "EXTERNAL_MAX_RETRY": 12,
+    "EXTERNAL_RETRY_INTERVAL": 13,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
+    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
+    "INDEXING_BLOCKS_AMOUNT": 14,
+    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
+    "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
+    "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
+  },
+  "CORE_RPC": {
+    "HOST": "__CORE_RPC_HOST__",
+    "PORT": 15,
+    "USERNAME": "__CORE_RPC_USERNAME__",
+    "PASSWORD": "__CORE_RPC_PASSWORD__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRUM_HOST__",
+    "PORT": 16,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "__SECOND_CORE_RPC_HOST__",
+    "PORT": 17,
+    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+  },
+  "DATABASE": {
+    "ENABLED": false,
+    "HOST": "__DATABASE_HOST__",
+    "SOCKET": "__DATABASE_SOCKET__",
+    "PORT": 18,
+    "DATABASE": "__DATABASE_DATABASE__",
+    "USERNAME": "__DATABASE_USERNAME__",
+    "PASSWORD": "__DATABASE_PASSWORD__"
+  },
+  "SYSLOG": {
+    "ENABLED": false,
+    "HOST": "__SYSLOG_HOST__",
+    "PORT": 19,
+    "MIN_PRIORITY": "__SYSLOG_MIN_PRIORITY__",
+    "FACILITY": "__SYSLOG_FACILITY__"
+  },
+  "STATISTICS": {
+    "ENABLED": false,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 20
+  },
+  "BISQ": {
+    "ENABLED": true,
+    "DATA_PATH": "__BISQ_DATA_PATH__"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": true,
+    "USE_ONION": true,
+    "HOST": "__SOCKS5PROXY_HOST__",
+    "PORT": "__SOCKS5PROXY_PORT__",
+    "USERNAME": "__SOCKS5PROXY_USERNAME__",
+    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
+    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
+  },
+  "LIGHTNING": {
+    "ENABLED": "__LIGHTNING_ENABLED__",
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__"
+  },
+  "LND": {
+    "TLS_CERT_PATH": "",
+    "MACAROON_PATH": "",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  }
+}

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -0,0 +1,62 @@
+import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+
+describe('Mempool Difficulty Adjustment', () => {
+  test('should calculate Difficulty Adjustments properly', () => {
+    const dt = (dtString) => {
+      return Math.floor(new Date(dtString).getTime() / 1000);
+    };
+
+    const vectors = [
+      [ // Vector 1
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // If not testnet, not used
+        ],
+        { // Expected Result
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: 0,
+        },
+      ],
+      [ // Vector 2 (testnet)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'testnet',                      // Network
+          dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result is same other than timeOffset
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
+          // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
+        },
+      ],
+    ] as [[number, number, number, number, string, number], DifficultyAdjustment][];
+
+    for (const vector of vectors) {
+      const result = calcDifficultyAdjustment(...vector[0]);
+      // previousRetarget is passed through untouched
+      expect(result.previousRetarget).toStrictEqual(vector[0][3]);
+      expect(result).toStrictEqual(vector[1]);
+    }
+  });
+});

backend/src/__tests__/config.test.ts

@@ -0,0 +1,143 @@
+import * as fs from 'fs';
+
+describe('Mempool Backend Config', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  test('should return defaults when no file is present', () => {
+    jest.isolateModules(() => {
+      jest.mock('../../mempool-config.json', () => ({}), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual({
+        ENABLED: true,
+        NETWORK: 'mainnet',
+        BACKEND: 'none',
+        BLOCKS_SUMMARIES_INDEXING: false,
+        HTTP_PORT: 8999,
+        SPAWN_CLUSTER_PROCS: 0,
+        API_URL_PREFIX: '/api/v1/',
+        AUTOMATIC_BLOCK_REINDEXING: false,
+        POLL_RATE_MS: 2000,
+        CACHE_DIR: './cache',
+        CLEAR_PROTECTION_MINUTES: 20,
+        RECOMMENDED_FEE_PERCENTILE: 50,
+        BLOCK_WEIGHT_UNITS: 4000000,
+        INITIAL_BLOCKS_AMOUNT: 8,
+        MEMPOOL_BLOCKS_AMOUNT: 8,
+        INDEXING_BLOCKS_AMOUNT: 11000,
+        PRICE_FEED_UPDATE_INTERVAL: 600,
+        USE_SECOND_NODE_FOR_MINFEE: false,
+        EXTERNAL_ASSETS: [],
+        EXTERNAL_MAX_RETRY: 1,
+        EXTERNAL_RETRY_INTERVAL: 0,
+        USER_AGENT: 'mempool',
+        STDOUT_LOG_MIN_PRIORITY: 'debug',
+        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+        ADVANCED_GBT_AUDIT: false,
+        ADVANCED_GBT_MEMPOOL: false,
+        TRANSACTION_INDEXING: false,
+      });
+
+      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
+
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+
+      expect(config.CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.DATABASE).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        SOCKET: '',
+        PORT: 3306,
+        DATABASE: 'mempool',
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SYSLOG).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        PORT: 514,
+        MIN_PRIORITY: 'info',
+        FACILITY: 'local7'
+      });
+
+      expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
+
+      expect(config.BISQ).toStrictEqual({ ENABLED: false, DATA_PATH: '/bisq/statsnode-data/btc_mainnet/db' });
+
+      expect(config.SOCKS5PROXY).toStrictEqual({
+        ENABLED: false,
+        USE_ONION: true,
+        HOST: '127.0.0.1',
+        PORT: 9050,
+        USERNAME: '',
+        PASSWORD: ''
+      });
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual({
+        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
+      });
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
+        MEMPOOL_API: 'https://mempool.space/api/v1',
+        MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+        LIQUID_API: 'https://liquid.network/api/v1',
+        LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+        BISQ_URL: 'https://bisq.markets/api',
+        BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+      });
+    });
+  });
+
+  test('should override the default values with the passed values', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      jest.mock('../../mempool-config.json', () => (fixture), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual(fixture.MEMPOOL);
+
+      expect(config.ELECTRUM).toStrictEqual(fixture.ELECTRUM);
+
+      expect(config.ESPLORA).toStrictEqual(fixture.ESPLORA);
+
+      expect(config.CORE_RPC).toStrictEqual(fixture.CORE_RPC);
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual(fixture.SECOND_CORE_RPC);
+
+      expect(config.DATABASE).toStrictEqual(fixture.DATABASE);
+
+      expect(config.SYSLOG).toStrictEqual(fixture.SYSLOG);
+
+      expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
+
+      expect(config.BISQ).toStrictEqual(fixture.BISQ);
+
+      expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+    });
+  });
+});

backend/src/api/audit.ts

@@ -0,0 +1,138 @@
+import config from '../config';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { Common } from './common';
+import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces';
+import blocksRepository from '../repositories/BlocksRepository';
+import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import blocks from '../api/blocks';
+
+const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
+
+class Audit {
+  auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
+   : { censored: string[], added: string[], fresh: string[], score: number } {
+    if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
+      return { censored: [], added: [], fresh: [], score: 0 };
+    }
+
+    const matches: string[] = []; // present in both mined block and template
+    const added: string[] = []; // present in mined block, not in template
+    const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const isCensored = {}; // missing, without excuse
+    const isDisplaced = {};
+    let displacedWeight = 0;
+
+    const inBlock = {};
+    const inTemplate = {};
+
+    const now = Math.round((Date.now() / 1000));
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+    // coinbase is always expected
+    if (transactions[0]) {
+      inTemplate[transactions[0].txid] = true;
+    }
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const txid of projectedBlocks[0].transactionIds) {
+      if (!inBlock[txid]) {
+        // tx is recent, may have reached the miner too late for inclusion
+        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          fresh.push(txid);
+        } else {
+          isCensored[txid] = true;
+        }
+        displacedWeight += mempool[txid].weight;
+      }
+      inTemplate[txid] = true;
+    }
+
+    displacedWeight += (4000 - transactions[0].weight);
+
+    // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
+    // these displaced transactions should occupy the first N weight units of the next projected block
+    let displacedWeightRemaining = displacedWeight;
+    let index = 0;
+    let lastFeeRate = Infinity;
+    let failures = 0;
+    while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
+      const txid = projectedBlocks[1].transactionIds[index];
+      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (fits || feeMatches) {
+        isDisplaced[txid] = true;
+        if (fits) {
+          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+        }
+        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+          displacedWeightRemaining -= mempool[txid].weight;
+        }
+        failures = 0;
+      } else {
+        failures++;
+      }
+      index++;
+    }
+
+    // mark unexpected transactions in the mined block as 'added'
+    let overflowWeight = 0;
+    let totalWeight = 0;
+    for (const tx of transactions) {
+      if (inTemplate[tx.txid]) {
+        matches.push(tx.txid);
+      } else {
+        if (!isDisplaced[tx.txid]) {
+          added.push(tx.txid);
+        } else {
+        }
+        let blockIndex = -1;
+        let index = -1;
+        projectedBlocks.forEach((block, bi) => {
+          const i = block.transactionIds.indexOf(tx.txid);
+          if (i >= 0) {
+            blockIndex = bi;
+            index = i;
+          }
+        });
+        overflowWeight += tx.weight;
+      }
+      totalWeight += tx.weight;
+    }
+
+    // transactions missing from near the end of our template are probably not being censored
+    let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
+    let maxOverflowRate = 0;
+    let rateThreshold = 0;
+    index = projectedBlocks[0].transactionIds.length - 1;
+    while (index >= 0) {
+      const txid = projectedBlocks[0].transactionIds[index];
+      if (overflowWeightRemaining > 0) {
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
+          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+        }
+      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+      }
+      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      index--;
+    }
+
+    const numCensored = Object.keys(isCensored).length;
+    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+
+    return {
+      censored: Object.keys(isCensored),
+      added,
+      fresh,
+      score
+    };
+  }
+}
+
+export default new Audit();

backend/src/api/backend-info.ts

@@ -1,60 +1,37 @@
-import * as fs from 'fs';
-import * as os from 'os';
-import logger from '../logger';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
-const { spawnSync } = require('child_process');
 
 class BackendInfo {
-  private gitCommitHash = '';
-  private hostname = '';
-  private version = '';
+  private backendInfo: IBackendInfo;
 
   constructor() {
-    this.setLatestCommitHash();
-    this.setVersion();
-    this.hostname = os.hostname();
-  }
-
-  public getBackendInfo(): IBackendInfo {
-    return {
-      hostname: this.hostname,
-      gitCommit: this.gitCommitHash,
-      version: this.version,
+    // This file is created by ./fetch-version.ts during building
+    const versionFile = path.join(__dirname, 'version.json')
+    var versionInfo;
+    if (fs.existsSync(versionFile)) {
+      versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
+    } else {
+      // Use dummy values if `versionFile` doesn't exist (e.g., during testing)
+      versionInfo = {
+        version: '?',
+        gitCommit: '?'
+      };
+    }
+    this.backendInfo = {
+      hostname: os.hostname(),
+      version: versionInfo.version,
+      gitCommit: versionInfo.gitCommit
     };
   }
 
+  public getBackendInfo(): IBackendInfo {
+    return this.backendInfo;
+  }
+
   public getShortCommitHash() {
-    return this.gitCommitHash.slice(0, 7);
-  }
-
-  private setLatestCommitHash(): void {
-    //TODO: share this logic with `generate-config.js`
-    if (process.env.DOCKER_COMMIT_HASH) {
-      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
-    } else {
-      try {
-        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
-        if (!gitRevParse.error) {
-          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
-          this.gitCommitHash = output ? output : '?';
-        } else if (gitRevParse.error.code === 'ENOENT') {
-          console.log('git not found, cannot parse git hash');
-          this.gitCommitHash = '?';
-        }
-      } catch (e: any) {
-        console.log('Could not load git commit info: ' + e.message);
-        this.gitCommitHash = '?';
-      }
-    }
-  }
-
-  private setVersion(): void {
-    try {
-      const packageJson = fs.readFileSync('package.json').toString();
-      this.version = JSON.parse(packageJson).version;
-    } catch (e) {
-      throw new Error(e instanceof Error ? e.message : 'Error');
-    }
+    return this.backendInfo.gitCommit.slice(0, 7);
   }
 }
 

backend/src/api/bisq/bisq.routes.ts

@@ -0,0 +1,381 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import { RequiredSpec } from '../../mempool.interfaces';
+import bisq from './bisq';
+import { MarketsApiError } from './interfaces';
+import marketsApi from './markets-api';
+
+class BisqRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', this.getBisqStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', this.getBisqTransaction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', this.getBisqBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', this.getBisqTip)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', this.getBisqBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', this.getBisqAddress)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', this.getBisqTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', this.getBisqMarketCurrencies.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', this.getBisqMarketDepth.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', this.getBisqMarketHloc.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', this.getBisqMarketMarkets.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', this.getBisqMarketOffers.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', this.getBisqMarketTicker.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', this.getBisqMarketTrades.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', this.getBisqMarketVolumes.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', this.getBisqMarketVolumes7d.bind(this))
+    ;
+  }
+
+
+  private getBisqStats(req: Request, res: Response) {
+    const result = bisq.getStats();
+    res.json(result);
+  }
+
+  private getBisqTip(req: Request, res: Response) {
+    const result = bisq.getLatestBlockHeight();
+    res.type('text/plain');
+    res.send(result.toString());
+  }
+
+  private getBisqTransaction(req: Request, res: Response) {
+    const result = bisq.getTransaction(req.params.txId);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Bisq transaction not found');
+    }
+  }
+
+  private getBisqTransactions(req: Request, res: Response) {
+    const types: string[] = [];
+    req.query.types = req.query.types || [];
+    if (!Array.isArray(req.query.types)) {
+      res.status(500).send('Types is not an array');
+      return;
+    }
+
+    for (const _type in req.query.types) {
+      if (typeof req.query.types[_type] === 'string') {
+        types.push(req.query.types[_type].toString());
+      }
+    }
+
+    const index = parseInt(req.params.index, 10) || 0;
+    const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25;
+    const [transactions, count] = bisq.getTransactions(index, length, types);
+    res.header('X-Total-Count', count.toString());
+    res.json(transactions);
+  }
+
+  private getBisqBlock(req: Request, res: Response) {
+    const result = bisq.getBlock(req.params.hash);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Bisq block not found');
+    }
+  }
+
+  private getBisqBlocks(req: Request, res: Response) {
+    const index = parseInt(req.params.index, 10) || 0;
+    const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25;
+    const [transactions, count] = bisq.getBlocks(index, length);
+    res.header('X-Total-Count', count.toString());
+    res.json(transactions);
+  }
+
+  private getBisqAddress(req: Request, res: Response) {
+    const result = bisq.getAddress(req.params.address.substr(1));
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Bisq address not found');
+    }
+  }
+
+  private getBisqMarketCurrencies(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'type': {
+        required: false,
+        types: ['crypto', 'fiat', 'all']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getCurrencies(p.type);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketCurrencies error'));
+    }
+  }
+
+  private getBisqMarketDepth(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getDepth(p.market);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketDepth error'));
+    }
+  }
+
+  private getBisqMarketMarkets(req: Request, res: Response) {
+    const result = marketsApi.getMarkets();
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketMarkets error'));
+    }
+  }
+
+  private getBisqMarketTrades(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+      'timestamp_from': {
+        required: false,
+        types: ['@number']
+      },
+      'timestamp_to': {
+        required: false,
+        types: ['@number']
+      },
+      'trade_id_to': {
+        required: false,
+        types: ['@string']
+      },
+      'trade_id_from': {
+        required: false,
+        types: ['@string']
+      },
+      'direction': {
+        required: false,
+        types: ['buy', 'sell']
+      },
+      'limit': {
+        required: false,
+        types: ['@number']
+      },
+      'sort': {
+        required: false,
+        types: ['asc', 'desc']
+      }
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getTrades(p.market, p.timestamp_from,
+      p.timestamp_to, p.trade_id_from, p.trade_id_to, p.direction, p.limit, p.sort);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTrades error'));
+    }
+  }
+
+  private getBisqMarketOffers(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+      'direction': {
+        required: false,
+        types: ['buy', 'sell']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getOffers(p.market, p.direction);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketOffers error'));
+    }
+  }
+
+  private getBisqMarketVolumes(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: false,
+        types: ['@string']
+      },
+      'interval': {
+        required: false,
+        types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto']
+      },
+      'timestamp_from': {
+        required: false,
+        types: ['@number']
+      },
+      'timestamp_to': {
+        required: false,
+        types: ['@number']
+      },
+      'milliseconds': {
+        required: false,
+        types: ['@boolean']
+      },
+      'timestamp': {
+        required: false,
+        types: ['no', 'yes']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getVolumes(p.market, p.timestamp_from, p.timestamp_to, p.interval, p.milliseconds, p.timestamp);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes error'));
+    }
+  }
+
+  private getBisqMarketHloc(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+      'interval': {
+        required: false,
+        types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto']
+      },
+      'timestamp_from': {
+        required: false,
+        types: ['@number']
+      },
+      'timestamp_to': {
+        required: false,
+        types: ['@number']
+      },
+      'milliseconds': {
+        required: false,
+        types: ['@boolean']
+      },
+      'timestamp': {
+        required: false,
+        types: ['no', 'yes']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to, p.milliseconds, p.timestamp);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketHloc error'));
+    }
+  }
+
+  private getBisqMarketTicker(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: false,
+        types: ['@string']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getTicker(p.market);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTicker error'));
+    }
+  }
+
+  private getBisqMarketVolumes7d(req: Request, res: Response) {
+    const result = marketsApi.getVolumesByTime(604800);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes7d error'));
+    }
+  }
+
+  private parseRequestParameters(requestParams: object, params: RequiredSpec): { [name: string]: any; } {
+    const final = {};
+    for (const i in params) {
+      if (params.hasOwnProperty(i)) {
+        if (params[i].required && requestParams[i] === undefined) {
+          return { error: i + ' parameter missing'};
+        }
+        if (typeof requestParams[i] === 'string') {
+          const str = (requestParams[i] || '').toString().toLowerCase();
+          if (params[i].types.indexOf('@number') > -1) {
+            const number = parseInt((str).toString(), 10);
+            final[i] = number;
+          } else if (params[i].types.indexOf('@string') > -1) {
+            final[i] = str;
+          } else if (params[i].types.indexOf('@boolean') > -1) {
+            final[i] = str === 'true' || str === 'yes';
+          } else if (params[i].types.indexOf(str) > -1) {
+            final[i] = str;
+          } else {
+            return { error: i + ' parameter invalid'};
+          }
+        } else if (typeof requestParams[i] === 'number') {
+          final[i] = requestParams[i];
+        }
+      }
+    }
+    return final;
+  }
+
+  private getBisqMarketErrorResponse(message: string): MarketsApiError {
+    return {
+      'success': 0,
+      'error': message
+    };
+  }
+
+}
+
+export default new BisqRoutes;

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -3,16 +3,21 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getTransactionHex(txId: string): Promise<string>;
   $getBlockHeightTip(): Promise<number>;
+  $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
+  $getRawBlock(hash: string): Promise<Buffer>;
   $getAddress(address: string): Promise<IEsploraApi.Address>;
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];
   $sendRawTransaction(rawTransaction: string): Promise<string>;
+  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
+  $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
 }
 export interface BitcoinRpcCredentials {
   host: string;

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -73,6 +73,14 @@ export namespace IBitcoinApi {
     time: number;                    //  (numeric) Same as blocktime
   }
 
+  export interface VerboseBlock extends Block {
+    tx: VerboseTransaction[];        // The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 "tx" result
+  }
+
+  export interface VerboseTransaction extends Transaction {
+    fee?: number;                   //  (numeric) The transaction fee in BTC, omitted if block undo data is not available
+  }
+
   export interface Vin {
     txid?: string;                   //  (string) The transaction id
     vout?: number;                   //  (string)

backend/src/api/bitcoin/bitcoin-api.ts

@@ -57,6 +57,11 @@ class BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
+  $getTransactionHex(txId: string): Promise<string> {
+    return this.$getRawTransaction(txId, true)
+      .then((tx) => tx.hex || '');
+  }
+
   $getBlockHeightTip(): Promise<number> {
     return this.bitcoindClient.getChainTips()
       .then((result: IBitcoinApi.ChainTips[]) => {
@@ -64,13 +69,21 @@ class BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
+  $getBlockHashTip(): Promise<string> {
+    return this.bitcoindClient.getChainTips()
+      .then((result: IBitcoinApi.ChainTips[]) => {
+        return result.find(tip => tip.status === 'active')!.hash;
+      });
+  }
+
   $getTxIdsForBlock(hash: string): Promise<string[]> {
     return this.bitcoindClient.getBlock(hash, 1)
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
-  $getRawBlock(hash: string): Promise<string> {
-    return this.bitcoindClient.getBlock(hash, 0);
+  $getRawBlock(hash: string): Promise<Buffer> {
+    return this.bitcoindClient.getBlock(hash, 0)
+      .then((raw: string) => Buffer.from(raw, "hex"));
   }
 
   $getBlockHash(height: number): Promise<string> {
@@ -123,6 +136,16 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.sendRawTransaction(rawTransaction);
   }
 
+  async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
+    const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
+    return {
+      spent: txOut === null,
+      status: {
+        confirmed: true,
+      }
+    };
+  }
+
   async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
     const outSpends: IEsploraApi.Outspend[] = [];
     const tx = await this.$getRawTransaction(txId, true, false);
@@ -141,6 +164,15 @@ class BitcoinApi implements AbstractBitcoinApi {
     return outSpends;
   }
 
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
   $getEstimatedHashrate(blockHeight: number): Promise<number> {
     // 120 is the default block span in Core
     return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
@@ -179,7 +211,9 @@ class BitcoinApi implements AbstractBitcoinApi {
         sequence: vin.sequence,
         txid: vin.txid || '',
         vout: vin.vout || 0,
-        witness: vin.txinwitness,
+        witness: vin.txinwitness || [],
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
       };
     });
 

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -0,0 +1,648 @@
+import { Application, Request, Response } from 'express';
+import axios from 'axios';
+import * as bitcoinjs from 'bitcoinjs-lib';
+import config from '../../config';
+import websocketHandler from '../websocket-handler';
+import mempool from '../mempool';
+import feeApi from '../fee-api';
+import mempoolBlocks from '../mempool-blocks';
+import bitcoinApi from './bitcoin-api-factory';
+import { Common } from '../common';
+import backendInfo from '../backend-info';
+import transactionUtils from '../transaction-utils';
+import { IEsploraApi } from './esplora-api.interface';
+import loadingIndicators from '../loading-indicators';
+import { TransactionExtended } from '../../mempool.interfaces';
+import logger from '../../logger';
+import blocks from '../blocks';
+import bitcoinClient from './bitcoin-client';
+import difficultyAdjustment from '../difficulty-adjustment';
+import transactionRepository from '../../repositories/TransactionRepository';
+
+class BitcoinRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', this.getBlocks.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      ;
+
+      if (config.MEMPOOL.BACKEND !== 'esplora') {
+        app
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', this.getMempool)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', this.getMempoolTxIds)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', this.getRecentMempoolTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', this.getTransaction)
+          .post(config.MEMPOOL.API_URL_PREFIX + 'tx', this.$postTransaction)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', this.getBlockTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', this.getBlockTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
+          ;
+      }
+  }
+
+
+  private getInitData(req: Request, res: Response) {
+    try {
+      const result = websocketHandler.getInitData();
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private getRecommendedFees(req: Request, res: Response) {
+    if (!mempool.isInSync()) {
+      res.statusCode = 503;
+      res.send('Service Unavailable');
+      return;
+    }
+    const result = feeApi.getRecommendedFee();
+    res.json(result);
+  }
+
+  private getMempoolBlocks(req: Request, res: Response) {
+    try {
+      const result = mempoolBlocks.getMempoolBlocks();
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private getTransactionTimes(req: Request, res: Response) {
+    if (!Array.isArray(req.query.txId)) {
+      res.status(500).send('Not an array');
+      return;
+    }
+    const txIds: string[] = [];
+    for (const _txId in req.query.txId) {
+      if (typeof req.query.txId[_txId] === 'string') {
+        txIds.push(req.query.txId[_txId].toString());
+      }
+    }
+
+    const times = mempool.getFirstSeenForTransactions(txIds);
+    res.json(times);
+  }
+
+  private async $getBatchedOutspends(req: Request, res: Response) {
+    if (!Array.isArray(req.query.txId)) {
+      res.status(500).send('Not an array');
+      return;
+    }
+    if (req.query.txId.length > 50) {
+      res.status(400).send('Too many txids requested');
+      return;
+    }
+    const txIds: string[] = [];
+    for (const _txId in req.query.txId) {
+      if (typeof req.query.txId[_txId] === 'string') {
+        txIds.push(req.query.txId[_txId].toString());
+      }
+    }
+
+    try {
+      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds);
+      res.json(batchedOutspends);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getCpfpInfo(req: Request, res: Response) {
+    if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
+      res.status(501).send(`Invalid transaction ID.`);
+      return;
+    }
+
+    const tx = mempool.getMempool()[req.params.txId];
+    if (tx) {
+      if (tx?.cpfpChecked) {
+        res.json({
+          ancestors: tx.ancestors,
+          bestDescendant: tx.bestDescendant || null,
+          descendants: tx.descendants || null,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+        });
+        return;
+      }
+
+      const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+
+      res.json(cpfpInfo);
+      return;
+    } else {
+      const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      if (cpfpInfo) {
+        res.json(cpfpInfo);
+        return;
+      }
+    }
+    res.status(404).send(`Transaction has no CPFP info available.`);
+  }
+
+  private getBackendInfo(req: Request, res: Response) {
+    res.json(backendInfo.getBackendInfo());
+  }
+
+  private async getTransaction(req: Request, res: Response) {
+    try {
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
+      res.json(transaction);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+      }
+      res.status(statusCode).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRawTransaction(req: Request, res: Response) {
+    try {
+      const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true);
+      res.setHeader('content-type', 'text/plain');
+      res.send(transaction.hex);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+      }
+      res.status(statusCode).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  /**
+   * Takes the PSBT as text/plain body, parses it, and adds the full
+   * parent transaction to each input that doesn't already have it.
+   * This is used for BTCPayServer / Trezor users which need access to
+   * the full parent transaction even with segwit inputs.
+   * It will respond with a text/plain PSBT in the same format (hex|base64).
+   */
+  private async postPsbtCompletion(req: Request, res: Response): Promise<void> {
+    res.setHeader('content-type', 'text/plain');
+    const notFoundError = `Couldn't get transaction hex for parent of input`;
+    try {
+      let psbt: bitcoinjs.Psbt;
+      let format: 'hex' | 'base64';
+      let isModified = false;
+      try {
+        psbt = bitcoinjs.Psbt.fromBase64(req.body);
+        format = 'base64';
+      } catch (e1) {
+        try {
+          psbt = bitcoinjs.Psbt.fromHex(req.body);
+          format = 'hex';
+        } catch (e2) {
+          throw new Error(`Unable to parse PSBT`);
+        }
+      }
+      for (const [index, input] of psbt.data.inputs.entries()) {
+        if (!input.nonWitnessUtxo) {
+          // Buffer.from ensures it won't be modified in place by reverse()
+          const txid = Buffer.from(psbt.txInputs[index].hash)
+            .reverse()
+            .toString('hex');
+
+          let transactionHex: string;
+          // If missing transaction, return 404 status error
+          try {
+            transactionHex = await bitcoinApi.$getTransactionHex(txid);
+            if (!transactionHex) {
+              throw new Error('');
+            }
+          } catch (err) {
+            throw new Error(`${notFoundError} #${index} @ ${txid}`);
+          }
+
+          psbt.updateInput(index, {
+            nonWitnessUtxo: Buffer.from(transactionHex, 'hex'),
+          });
+          if (!isModified) {
+            isModified = true;
+          }
+        }
+      }
+      if (isModified) {
+        res.send(format === 'hex' ? psbt.toHex() : psbt.toBase64());
+      } else {
+        // Not modified
+        // 422 Unprocessable Entity
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
+        res.status(422).send(`Psbt had no missing nonWitnessUtxos.`);
+      }
+    } catch (e: any) {
+      if (e instanceof Error && new RegExp(notFoundError).test(e.message)) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async getTransactionStatus(req: Request, res: Response) {
+    try {
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
+      res.json(transaction.status);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+      }
+      res.status(statusCode).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getStrippedBlockTransactions(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlock(req: Request, res: Response) {
+    try {
+      const block = await blocks.$getBlock(req.params.hash);
+
+      const blockAge = new Date().getTime() / 1000 - block.timestamp;
+      const day = 24 * 3600;
+      let cacheDuration;
+      if (blockAge > 365 * day) {
+        cacheDuration = 30 * day;
+      } else if (blockAge > 30 * day) {
+        cacheDuration = 10 * day;
+      } else {
+        cacheDuration = 600
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString());
+      res.json(block);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockHeader(req: Request, res: Response) {
+    try {
+      const blockHeader = await bitcoinApi.$getBlockHeader(req.params.hash);
+      res.setHeader('content-type', 'text/plain');
+      res.send(blockHeader);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockAuditSummary(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getBlockAuditSummary(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlocks(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+        const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+        res.json(await blocks.$getBlocks(height, 15));
+      } else { // Liquid, Bisq
+        return await this.getLegacyBlocks(req, res);
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getLegacyBlocks(req: Request, res: Response) {
+    try {
+      const returnBlocks: IEsploraApi.Block[] = [];
+      const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();
+
+      // Check if block height exist in local cache to skip the hash lookup
+      const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight);
+      let startFromHash: string | null = null;
+      if (blockByHeight) {
+        startFromHash = blockByHeight.id;
+      } else {
+        startFromHash = await bitcoinApi.$getBlockHash(fromHeight);
+      }
+
+      let nextHash = startFromHash;
+      for (let i = 0; i < 10 && nextHash; i++) {
+        const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
+        if (localBlock) {
+          returnBlocks.push(localBlock);
+          nextHash = localBlock.previousblockhash;
+        } else {
+          const block = await bitcoinApi.$getBlock(nextHash);
+          returnBlocks.push(block);
+          nextHash = block.previousblockhash;
+        }
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(returnBlocks);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+  
+  private async getBlockTransactions(req: Request, res: Response) {
+    try {
+      loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
+
+      const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
+      const transactions: TransactionExtended[] = [];
+      const startingIndex = Math.max(0, parseInt(req.params.index || '0', 10));
+
+      const endIndex = Math.min(startingIndex + 10, txIds.length);
+      for (let i = startingIndex; i < endIndex; i++) {
+        try {
+          const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true);
+          transactions.push(transaction);
+          loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100);
+        } catch (e) {
+          logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e));
+        }
+      }
+      res.json(transactions);
+    } catch (e) {
+      loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100);
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockHeight(req: Request, res: Response) {
+    try {
+      const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10));
+      res.send(blockHash);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getAddress(req: Request, res: Response) {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+
+    try {
+      const addressData = await bitcoinApi.$getAddress(req.params.address);
+      res.json(addressData);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        return res.status(413).send(e instanceof Error ? e.message : e);
+      }
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getAddressTransactions(req: Request, res: Response) {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+
+    try {
+      const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId);
+      res.json(transactions);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        return res.status(413).send(e instanceof Error ? e.message : e);
+      }
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getAdressTxChain(req: Request, res: Response) {
+    res.status(501).send('Not implemented');
+  }
+
+  private async getAddressPrefix(req: Request, res: Response) {
+    try {
+      const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix);
+      res.send(blockHash);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRecentMempoolTransactions(req: Request, res: Response) {
+    const latestTransactions = Object.entries(mempool.getMempool())
+      .sort((a, b) => (b[1].firstSeen || 0) - (a[1].firstSeen || 0))
+      .slice(0, 10).map((tx) => Common.stripTransaction(tx[1]));
+
+    res.json(latestTransactions);
+  }
+
+  private async getMempool(req: Request, res: Response) {
+    const info = mempool.getMempoolInfo();
+    res.json({
+      count: info.size,
+      vsize: info.bytes,
+      total_fee: info.total_fee * 1e8,
+      fee_histogram: []
+    });
+  }
+
+  private async getMempoolTxIds(req: Request, res: Response) {
+    try {
+      const rawMempool = await bitcoinApi.$getRawMempool();
+      res.send(rawMempool);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockTipHeight(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getBlockHeightTip();
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockTipHash(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getBlockHashTip();
+      res.setHeader('content-type', 'text/plain');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRawBlock(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getRawBlock(req.params.hash);
+      res.setHeader('content-type', 'application/octet-stream');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getTxIdsForBlock(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async validateAddress(req: Request, res: Response) {
+    try {
+      const result = await bitcoinClient.validateAddress(req.params.address);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getTransactionOutspends(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getOutspends(req.params.txId);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private getDifficultyChange(req: Request, res: Response) {
+    try {
+      const da = difficultyAdjustment.getDifficultyAdjustment();
+      if (da) {
+        res.json(da);
+      } else {
+        res.status(503).send(`Service Temporarily Unavailable`);
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $postTransaction(req: Request, res: Response) {
+    res.setHeader('content-type', 'text/plain');
+    try {
+      let rawTx;
+      if (typeof req.body === 'object') {
+        rawTx = Object.keys(req.body)[0];
+      } else {
+        rawTx = req.body;
+      }
+      const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx);
+      res.send(txIdResult);
+    } catch (e: any) {
+      res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
+        : (e.message || 'Error'));
+    }
+  }
+
+  private async $postTransactionForm(req: Request, res: Response) {
+    res.setHeader('content-type', 'text/plain');
+    const matches = /tx=([a-z0-9]+)/.exec(req.body);
+    let txHex = '';
+    if (matches && matches[1]) {
+      txHex = matches[1];
+    }
+    try {
+      const txIdResult = await bitcoinClient.sendRawTransaction(txHex);
+      res.send(txIdResult);
+    } catch (e: any) {
+      res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
+        : (e.message || 'Error'));
+    }
+  }
+
+}
+
+export default new BitcoinRoutes();

backend/src/api/bitcoin/electrum-api.ts

@@ -1,12 +1,11 @@
 import config from '../../config';
+import Client from '@mempool/electrum-client';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import { IElectrumApi } from './electrum-api.interface';
 import BitcoinApi from './bitcoin-api';
 import logger from '../../logger';
-import * as ElectrumClient from '@mempool/electrum-client';
-import * as sha256 from 'crypto-js/sha256';
-import * as hexEnc from 'crypto-js/enc-hex';
+import crypto from "crypto-js";
 import loadingIndicators from '../loading-indicators';
 import memoryCache from '../memory-cache';
 
@@ -26,7 +25,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       onLog: (str) => { logger.debug(str); },
     };
 
-    this.electrumClient = new ElectrumClient(
+    this.electrumClient = new Client(
       config.ELECTRUM.PORT,
       config.ELECTRUM.HOST,
       config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
@@ -35,7 +34,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     );
 
     this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
-      .then(() => {})
+      .then(() => { })
       .catch((err) => {
         logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
       });
@@ -95,7 +94,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-     return [];
+      return [];
     }
 
     try {
@@ -144,8 +143,8 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   }
 
   private encodeScriptHash(scriptPubKey: string): string {
-    const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey)));
-    return addrScripthash.match(/.{2}/g).reverse().join('');
+    const addrScripthash = crypto.enc.Hex.stringify(crypto.SHA256(crypto.enc.Hex.parse(scriptPubKey)));
+    return addrScripthash!.match(/.{2}/g)!.reverse().join('');
   }
 
 }

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -25,10 +25,10 @@ export namespace IEsploraApi {
     is_coinbase: boolean;
     scriptsig: string;
     scriptsig_asm: string;
-    inner_redeemscript_asm?: string;
-    inner_witnessscript_asm?: string;
+    inner_redeemscript_asm: string;
+    inner_witnessscript_asm: string;
     sequence: any;
-    witness?: string[];
+    witness: string[];
     prevout: Vout | null;
     // Elements
     is_pegin?: boolean;

backend/src/api/bitcoin/esplora-api.ts

@@ -20,11 +20,21 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
+  $getTransactionHex(txId: string): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
   $getBlockHeightTip(): Promise<number> {
     return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
       .then((response) => response.data);
   }
 
+  $getBlockHashTip(): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
   $getTxIdsForBlock(hash: string): Promise<string[]> {
     return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
@@ -45,6 +55,11 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
+  $getRawBlock(hash: string): Promise<Buffer> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+      .then((response) => { return Buffer.from(response.data); });
+  }
+
   $getAddress(address: string): Promise<IEsploraApi.Address> {
     throw new Error('Method getAddress not implemented.');
   }
@@ -61,8 +76,23 @@ class ElectrsApi implements AbstractBitcoinApi {
     throw new Error('Method not implemented.');
   }
 
-  $getOutspends(): Promise<IEsploraApi.Outspend[]> {
-    throw new Error('Method not implemented.');
+  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
+    return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
   }
 }
 

backend/src/api/blocks.ts

@@ -2,11 +2,12 @@ import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
 import bitcoinClient from './bitcoin/bitcoin-client';
+import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
@@ -17,16 +18,26 @@ import BlocksRepository from '../repositories/BlocksRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
 import fiatConversion from './fiat-conversion';
-import RatesRepository from '../repositories/RatesRepository';
 import poolsParser from './pools-parser';
+import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import cpfpRepository from '../repositories/CpfpRepository';
+import transactionRepository from '../repositories/TransactionRepository';
+import mining from './mining/mining';
+import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import PricesRepository from '../repositories/PricesRepository';
+import priceUpdater from '../tasks/price-updater';
+import { Block } from 'bitcoinjs-lib';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
+  private blockSummaries: BlockSummary[] = [];
   private currentBlockHeight = 0;
   private currentDifficulty = 0;
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
+  private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
   constructor() { }
 
@@ -38,10 +49,22 @@ class Blocks {
     this.blocks = blocks;
   }
 
+  public getBlockSummaries(): BlockSummary[] {
+    return this.blockSummaries;
+  }
+
+  public setBlockSummaries(blockSummaries: BlockSummary[]) {
+    this.blockSummaries = blockSummaries;
+  }
+
   public setNewBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void) {
     this.newBlockCallbacks.push(fn);
   }
 
+  public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>) {
+    this.newAsyncBlockCallbacks.push(fn);
+  }
+
   /**
    * Return the list of transaction for a block
    * @param blockHash
@@ -106,6 +129,27 @@ class Blocks {
     return transactions;
   }
 
+  /**
+   * Return a block summary (list of stripped transactions)
+   * @param block
+   * @returns BlockSummary
+   */
+  private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
+    const stripped = block.tx.map((tx) => {
+      return {
+        txid: tx.txid,
+        vsize: tx.weight / 4,
+        fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
+        value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
+      };
+    });
+
+    return {
+      id: block.hash,
+      transactions: stripped
+    };
+  }
+
   /**
    * Return a block with additional data (reward, coinbase, fees...)
    * @param block
@@ -117,6 +161,7 @@ class Blocks {
     blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
     blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
     blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
+    blockExtended.extras.usd = fiatConversion.getConversionRates().USD;
 
     if (block.height === 0) {
       blockExtended.extras.medianFee = 0; // 50th percentiles
@@ -135,7 +180,7 @@ class Blocks {
       blockExtended.extras.avgFeeRate = stats.avgfeerate;
     }
 
-    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
       if (blockExtended.extras?.coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
@@ -150,14 +195,18 @@ class Blocks {
       if (!pool) { // We should never have this situation in practise
         logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
-        return blockExtended;
+      } else {
+        blockExtended.extras.pool = {
+          id: pool.id,
+          name: pool.name,
+          slug: pool.slug,
+        };
       }
 
-      blockExtended.extras.pool = {
-        id: pool.id,
-        name: pool.name,
-        slug: pool.slug,
-      };
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
+      if (auditScore != null) {
+        blockExtended.extras.matchRate = auditScore.matchRate;
+      }
     }
 
     return blockExtended;
@@ -212,15 +261,121 @@ class Blocks {
   }
 
   /**
-   * [INDEXING] Index all blocks metadata for the mining dashboard
+   * [INDEXING] Index all blocks summaries for the block txs visualization
    */
-  public async $generateBlockDatabase() {
-    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
-    if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
+  public async $generateBlocksSummariesDatabase(): Promise<void> {
+    if (Common.blocksSummariesIndexingEnabled() === false) {
       return;
     }
 
     try {
+      // Get all indexed block hash
+      const indexedBlocks = await blocksRepository.$getIndexedBlocks();
+      const indexedBlockSummariesHashesArray = await BlocksSummariesRepository.$getIndexedSummariesId();
+
+      const indexedBlockSummariesHashes = {}; // Use a map for faster seek during the indexing loop
+      for (const hash of indexedBlockSummariesHashesArray) {
+        indexedBlockSummariesHashes[hash] = true;
+      }
+
+      // Logging
+      let newlyIndexed = 0;
+      let totalIndexed = indexedBlockSummariesHashesArray.length;
+      let indexedThisRun = 0;
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+
+      for (const block of indexedBlocks) {
+        if (indexedBlockSummariesHashes[block.hash] === true) {
+          continue;
+        }
+
+        // Logging
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 5) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
+          const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
+          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+        }
+
+        await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
+
+        // Logging
+        indexedThisRun++;
+        totalIndexed++;
+        newlyIndexed++;
+      }
+      if (newlyIndexed > 0) {
+        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+      } else {
+        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+      }
+    } catch (e) {
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Index transaction CPFP data for all blocks
+   */
+   public async $generateCPFPDatabase(): Promise<void> {
+    if (Common.cpfpIndexingEnabled() === false) {
+      return;
+    }
+
+    try {
+      // Get all indexed block hash
+      const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks();
+
+      if (!unindexedBlocks?.length) {
+        return;
+      }
+
+      // Logging
+      let count = 0;
+      let countThisRun = 0;
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+
+      for (const block of unindexedBlocks) {
+        // Logging
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds);
+          const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
+          logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          timer = new Date().getTime() / 1000;
+          countThisRun = 0;
+        }
+
+        await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block
+
+        // Logging
+        count++;
+        countThisRun++;
+      }
+      if (count > 0) {
+        logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
+      } else {
+        logger.debug(`CPFP indexing completed: indexed ${count} blocks`);
+      }
+    } catch (e) {
+      logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Index all blocks metadata for the mining dashboard
+   */
+  public async $generateBlockDatabase(): Promise<boolean> {
+    try {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
       let currentBlockHeight = blockchainInfo.blocks;
 
       let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
@@ -258,13 +413,12 @@ class Blocks {
           }
           ++indexedThisRun;
           ++totalIndexed;
-          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+          const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
           if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
             const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-            const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+            const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
             const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
-            const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds);
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`);
             timer = new Date().getTime() / 1000;
             indexedThisRun = 0;
             loadingIndicators.setProgress('block-indexing', progress, false);
@@ -280,18 +434,19 @@ class Blocks {
 
         currentBlockHeight -= chunkSize;
       }
-      logger.info(`Indexed ${newlyIndexed} blocks`);
+      if (newlyIndexed > 0) {
+        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+      } else {
+        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+      }
       loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
-      logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
       loadingIndicators.setProgress('block-indexing', 100);
-      return;
+      throw e;
     }
 
-    const chainValid = await BlocksRepository.$validateChain();
-    if (!chainValid) {
-      indexer.reindex();
-    }
+    return await BlocksRepository.$validateChain();
   }
 
   public async $updateBlocks() {
@@ -323,7 +478,7 @@ class Blocks {
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          const previousPeriodBlock = await bitcoinApi.$getBlock(previousPeriodBlockHash);
+          const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -341,10 +496,15 @@ class Blocks {
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
-      const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+      const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
+      const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+      const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
+
+      // start async callbacks
+      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
 
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
@@ -354,18 +514,55 @@ class Blocks {
             // We assume there won't be a reorg with more than 10 block depth
             await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
             await HashratesRepository.$deleteLastEntries();
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
             for (let i = 10; i >= 0; --i) {
-              await this.$indexBlock(lastBlock['height'] - i);
+              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
+              await this.$getStrippedBlockTransactions(newBlock.id, true, true);
+              if (config.MEMPOOL.TRANSACTION_INDEXING) {
+                await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
+              }
             }
+            await mining.$indexDifficultyAdjustments();
+            await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
+
+          const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
+            await blocksRepository.$saveBlockPrices([{
+              height: blockExtended.height,
+              priceId: lastestPriceId,
+            }]);
+          } else {
+            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
+            setTimeout(() => {
+              indexer.runSingleTask('blocksPrices');
+            }, 10000);
+          }
+
+          // Save blocks summary for visualization if it's enabled
+          if (Common.blocksSummariesIndexingEnabled() === true) {
+            await this.$getStrippedBlockTransactions(blockExtended.id, true);
+          }
+          if (config.MEMPOOL.TRANSACTION_INDEXING) {
+            this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
+          }
         }
       }
-      if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) {
-        await RatesRepository.$saveRate(blockExtended.height, fiatConversion.getConversionRates());
-      }
 
       if (block.height % 2016 === 0) {
+        if (Common.indexingEnabled()) {
+          await DifficultyAdjustmentsRepository.$saveAdjustments({
+            time: block.timestamp,
+            height: block.height,
+            difficulty: block.difficulty,
+            adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
+          });
+        }
+
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
@@ -375,6 +572,10 @@ class Blocks {
       if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
         this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
       }
+      this.blockSummaries.push(blockSummary);
+      if (this.blockSummaries.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
+        this.blockSummaries = this.blockSummaries.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
+      }
 
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
@@ -382,6 +583,9 @@ class Blocks {
       if (!memPool.hasPriority()) {
         diskCache.$saveCacheToDisk();
       }
+
+      // wait for pending async callbacks to finish
+      await Promise.all(callbackPromises);
     }
   }
 
@@ -422,13 +626,14 @@ class Blocks {
       }
     }
 
-    const block = await bitcoinApi.$getBlock(hash);
-
     // Not Bitcoin network, return the block as it
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return block;
+      return await bitcoinApi.$getBlock(hash);
     }
 
+    let block = await bitcoinClient.getBlock(hash);
+    block = prepareBlock(block);
+
     // Bitcoin network, add our custom data on top
     const transactions = await this.$getTransactionsExtended(hash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -440,48 +645,87 @@ class Blocks {
     return blockExtended;
   }
 
-  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-    try {
-      let currentHeight = fromHeight !== undefined ? fromHeight : this.getCurrentBlockHeight();
-      const returnBlocks: BlockExtended[] = [];
-
-      if (currentHeight < 0) {
-        return returnBlocks;
+  public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
+    skipDBLookup = false): Promise<TransactionStripped[]>
+  {
+    if (skipMemoryCache === false) {
+      // Check the memory cache
+      const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);
+      if (cachedSummary?.transactions?.length) {
+        return cachedSummary.transactions;
       }
-
-      if (currentHeight === 0 && Common.indexingEnabled()) {
-        currentHeight = await blocksRepository.$mostRecentBlockHeight();
-      }
-
-      // Check if block height exist in local cache to skip the hash lookup
-      const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
-      let startFromHash: string | null = null;
-      if (blockByHeight) {
-        startFromHash = blockByHeight.id;
-      } else if (!Common.indexingEnabled()) {
-        startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
-      }
-
-      let nextHash = startFromHash;
-      for (let i = 0; i < limit && currentHeight >= 0; i++) {
-        let block = this.getBlocks().find((b) => b.height === currentHeight);
-        if (block) {
-          returnBlocks.push(block);
-        } else if (Common.indexingEnabled()) {
-          block = await this.$indexBlock(currentHeight);
-          returnBlocks.push(block);
-        } else if (nextHash != null) {
-          block = prepareBlock(await bitcoinApi.$getBlock(nextHash));
-          nextHash = block.previousblockhash;
-          returnBlocks.push(block);
-        }
-        currentHeight--;
-      }
-
-      return returnBlocks;
-    } catch (e) {
-      throw e;
     }
+
+    // Check if it's indexed in db
+    if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) {
+      const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash);
+      if (indexedSummary !== undefined && indexedSummary?.transactions?.length) {
+        return indexedSummary.transactions;
+      }
+    }
+
+    // Call Core RPC
+    const block = await bitcoinClient.getBlock(hash, 2);
+    const summary = this.summarizeBlock(block);
+
+    // Index the response if needed
+    if (Common.blocksSummariesIndexingEnabled() === true) {
+      await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary});
+    }
+
+    return summary.transactions;
+  }
+
+  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
+    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+    const returnBlocks: BlockExtended[] = [];
+
+    if (currentHeight < 0) {
+      return returnBlocks;
+    }
+
+    // Check if block height exist in local cache to skip the hash lookup
+    const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
+    let startFromHash: string | null = null;
+    if (blockByHeight) {
+      startFromHash = blockByHeight.id;
+    } else if (!Common.indexingEnabled()) {
+      startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
+    }
+
+    let nextHash = startFromHash;
+    for (let i = 0; i < limit && currentHeight >= 0; i++) {
+      let block = this.getBlocks().find((b) => b.height === currentHeight);
+      if (block) {
+        returnBlocks.push(block);
+      } else if (Common.indexingEnabled()) {
+        block = await this.$indexBlock(currentHeight);
+        returnBlocks.push(block);
+      } else if (nextHash != null) {
+        block = prepareBlock(await bitcoinClient.getBlock(nextHash));
+        nextHash = block.previousblockhash;
+        returnBlocks.push(block);
+      }
+      currentHeight--;
+    }
+
+    return returnBlocks;
+  }
+
+  public async $getBlockAuditSummary(hash: string): Promise<any> {
+    let summary;
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+      summary = await BlocksAuditsRepository.$getBlockAudit(hash);
+    }
+
+    // fallback to non-audited transaction summary
+    if (!summary?.transactions?.length) {
+      const strippedTransactions = await this.$getStrippedBlockTransactions(hash);
+      summary = {
+        transactions: strippedTransactions
+      };
+    }
+    return summary;
   }
 
   public getLastDifficultyAdjustmentTime(): number {
@@ -495,6 +739,62 @@ class Blocks {
   public getCurrentBlockHeight(): number {
     return this.currentBlockHeight;
   }
+
+  public async $indexCPFP(hash: string, height: number): Promise<void> {
+    let transactions;
+    if (false/*Common.blocksSummariesIndexingEnabled()*/) {
+      transactions = await this.$getStrippedBlockTransactions(hash);
+      const rawBlock = await bitcoinApi.$getRawBlock(hash);
+      const block = Block.fromBuffer(rawBlock);
+      const txMap = {};
+      for (const tx of block.transactions || []) {
+        txMap[tx.getId()] = tx;
+      }
+      for (const tx of transactions) {
+        if (txMap[tx.txid]?.ins) {
+          tx.vin = txMap[tx.txid].ins.map(vin => {
+            return {
+              txid: vin.hash
+            };
+          });
+        }
+      }
+    } else {
+      const block = await bitcoinClient.getBlock(hash, 2);
+      transactions = block.tx.map(tx => {
+        tx.vsize = tx.weight / 4;
+        return tx;
+      });
+    }
+ 
+    let cluster: TransactionStripped[] = [];
+    let ancestors: { [txid: string]: boolean } = {};
+    for (let i = transactions.length - 1; i >= 0; i--) {
+      const tx = transactions[i];
+      if (!ancestors[tx.txid]) {
+        let totalFee = 0;
+        let totalVSize = 0;
+        cluster.forEach(tx => {
+          totalFee += tx?.fee || 0;
+          totalVSize += tx.vsize;
+        });
+        const effectiveFeePerVsize = (totalFee * 100_000_000) / totalVSize;
+        if (cluster.length > 1) {
+          await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: (tx.fee || 0) * 100_000_000 }; }), effectiveFeePerVsize);
+          for (const tx of cluster) {
+            await transactionRepository.$setCluster(tx.txid, cluster[0].txid);
+          }
+        }
+        cluster = [];
+        ancestors = {};
+      }
+      cluster.push(tx);
+      tx.vin.forEach(vin => {
+        ancestors[vin.txid] = true;
+      });
+    }
+    await blocksRepository.$setCPFPIndexed(hash);
+  }
 }
 
 export default new Blocks();

backend/src/api/common.ts

@@ -1,5 +1,7 @@
 import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
+import { NodeSocket } from '../repositories/NodesSocketsRepository';
+import { isIP } from 'net';
 export class Common {
   static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
     '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -114,7 +116,7 @@ export class Common {
       totalFees += tx.bestDescendant.fee;
     }
 
-    tx.effectiveFeePerVsize = Math.max(Common.isLiquid() ? 0.1 : 1, totalFees / (totalWeight / 4));
+    tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4));
     tx.cpfpChecked = true;
 
     return {
@@ -172,9 +174,136 @@ export class Common {
 
   static indexingEnabled(): boolean {
     return (
-      ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
+      ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
       config.DATABASE.ENABLED === true &&
       config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
     );
   }
+
+  static blocksSummariesIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
+    );
+  }
+
+  static cpfpIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.TRANSACTION_INDEXING === true
+    );
+  }
+
+  static setDateMidnight(date: Date): void {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+  }
+
+  static channelShortIdToIntegerId(channelId: string): string {
+    if (channelId.indexOf('x') === -1) { // Already an integer id
+      return channelId;
+    }
+    if (channelId.indexOf('/') !== -1) { // Topology import
+      channelId = channelId.slice(0, -2);
+    }
+    const s = channelId.split('x').map(part => BigInt(part));
+    return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
+  }
+
+  /** Decodes a channel id returned by lnd as uint64 to a short channel id */
+  static channelIntegerIdToShortId(id: string): string {
+    if (id.indexOf('/') !== -1) {
+      id = id.slice(0, -2);
+    }
+    
+    if (id.indexOf('x') !== -1) { // Already a short id
+      return id;
+    }
+
+    const n = BigInt(id);
+    return [
+      n >> 40n, // nth block
+      (n >> 16n) & 0xffffffn, // nth tx of the block
+      n & 0xffffn // nth output of the tx
+    ].join('x');
+  }
+
+  static utcDateToMysql(date?: number): string {
+    const d = new Date((date || 0) * 1000);
+    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
+  }
+
+  static findSocketNetwork(addr: string): {network: string | null, url: string} {
+    let network: string | null = null;
+    let url = addr.split('://')[1];
+
+    if (!url) {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    if (addr.indexOf('onion') !== -1) {
+      if (url.split('.')[0].length >= 56) {
+        network = 'torv3';
+      } else {
+        network = 'torv2';
+      }
+    } else if (addr.indexOf('i2p') !== -1) {
+      network = 'i2p';
+    } else if (addr.indexOf('ipv4') !== -1) {
+      const ipv = isIP(url.split(':')[0]);
+      if (ipv === 4) {
+        network = 'ipv4';
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else if (addr.indexOf('ipv6') !== -1) {
+      url = url.split('[')[1].split(']')[0];
+      const ipv = isIP(url);
+      if (ipv === 6) {
+        const parts = addr.split(':');
+        network = 'ipv6';
+        url = `[${url}]:${parts[parts.length - 1]}`;
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    return {
+      network: network,
+      url: url,
+    };
+  }
+
+  static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      return {
+        publicKey: publicKey,
+        network: socket.network,
+        addr: socket.addr,
+      };
+    } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
+      const formatted = this.findSocketNetwork(socket.addr);
+      return {
+        publicKey: publicKey,
+        network: formatted.network,
+        addr: formatted.url,
+      };
+    }
+  }
 }

backend/src/api/database-migration.ts

@@ -4,11 +4,25 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 19;
-  private queryTimeout = 120000;
+  private static currentVersion = 49;
+  private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
+  private uniqueLogs: string[] = [];
+
+  private blocksTruncatedMessage = `'blocks' table has been truncated.`;
+  private hashratesTruncatedMessage = `'hashrates' table has been truncated.`;
+
+  /**
+   * Avoid printing multiple time the same message
+   */
+  private uniqueLog(loggerFunction: any, msg: string) {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
 
-  constructor() { }
   /**
    * Entry point
    */
@@ -39,6 +53,16 @@ class DatabaseMigration {
       process.exit(-1);
     }
 
+    if (databaseSchemaVersion === 0) {
+      logger.info('Initializing database (first run, clean install)');
+    }
+
+    if (databaseSchemaVersion <= 2) {
+      // Disable some spam logs when they're not relevant
+      this.uniqueLogs.push(this.blocksTruncatedMessage);
+      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+    }
+
     logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
     logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
     if (databaseSchemaVersion >= DatabaseMigration.currentVersion) {
@@ -56,10 +80,13 @@ class DatabaseMigration {
     }
 
     if (DatabaseMigration.currentVersion > databaseSchemaVersion) {
-      logger.notice('MIGRATIONS: Upgrading database schema');
       try {
         await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
-        logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
+        if (databaseSchemaVersion === 0) {
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
+        } else {
+          logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
+        }
       } catch (e) {
         logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e);
       }
@@ -75,121 +102,345 @@ class DatabaseMigration {
     await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
 
     const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
-    try {
-      await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
-      await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
-      if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
-        await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
-      }
-      if (databaseSchemaVersion < 3) {
-        await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
-      }
-      if (databaseSchemaVersion < 4) {
-        await this.$executeQuery('DROP table IF EXISTS blocks;');
-        await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
-      }
-      if (databaseSchemaVersion < 5 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
-      }
 
-      if (databaseSchemaVersion < 6 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE blocks;');  // Need to re-index
-        // Cleanup original blocks fields type
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
-        // We also fix the pools.id type so we need to drop/re-create the foreign key
-        await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
-        await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
-        await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
-        // Add new block indexing fields
-        await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
-        await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
-      }
+    await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
+    await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
+    if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
+      await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
+      await this.updateToSchemaVersion(2);
+    }
+    if (databaseSchemaVersion < 3) {
+      await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+      await this.updateToSchemaVersion(3);
+    }
+    if (databaseSchemaVersion < 4) {
+      await this.$executeQuery('DROP table IF EXISTS blocks;');
+      await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
+      await this.updateToSchemaVersion(4);
+    }
+    if (databaseSchemaVersion < 5 && isBitcoin === true) {
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+      await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(5);
+    }
 
-      if (databaseSchemaVersion < 7 && isBitcoin === true) {
-        await this.$executeQuery('DROP table IF EXISTS hashrates;');
-        await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
-      }
+    if (databaseSchemaVersion < 6 && isBitcoin === true) {
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;');  // Need to re-index
+      // Cleanup original blocks fields type
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
+      // We also fix the pools.id type so we need to drop/re-create the foreign key
+      await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
+      await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
+      await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
+      // Add new block indexing fields
+      await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
+      await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
+      await this.updateToSchemaVersion(6);
+    }
 
-      if (databaseSchemaVersion < 8 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
-      }
+    if (databaseSchemaVersion < 7 && isBitcoin === true) {
+      await this.$executeQuery('DROP table IF EXISTS hashrates;');
+      await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
+      await this.updateToSchemaVersion(7);
+    }
 
-      if (databaseSchemaVersion < 9 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
-      }
+    if (databaseSchemaVersion < 8 && isBitcoin === true) {
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+      await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
+      await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
+      await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
+      await this.updateToSchemaVersion(8);
+    }
 
-      if (databaseSchemaVersion < 10 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
-      }
+    if (databaseSchemaVersion < 9 && isBitcoin === true) {
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
+      await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+      await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
+      await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
+      await this.updateToSchemaVersion(9);
+    }
 
-      if (databaseSchemaVersion < 11 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
-        await this.$executeQuery(`ALTER TABLE blocks
-          ADD avg_fee INT UNSIGNED NULL,
-          ADD avg_fee_rate INT UNSIGNED NULL
-        `);
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
-      }
+    if (databaseSchemaVersion < 10 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
+      await this.updateToSchemaVersion(10);
+    }
 
-      if (databaseSchemaVersion < 12 && isBitcoin === true) {
-        // No need to re-index because the new data type can contain larger values
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-      }
+    if (databaseSchemaVersion < 11 && isBitcoin === true) {
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+      await this.$executeQuery(`ALTER TABLE blocks
+        ADD avg_fee INT UNSIGNED NULL,
+        ADD avg_fee_rate INT UNSIGNED NULL
+      `);
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(11);
+    }
 
-      if (databaseSchemaVersion < 13 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-      }
+    if (databaseSchemaVersion < 12 && isBitcoin === true) {
+      // No need to re-index because the new data type can contain larger values
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(12);
+    }
 
-      if (databaseSchemaVersion < 14 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
-        await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
-      }
+    if (databaseSchemaVersion < 13 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(13);
+    }
 
-      if (databaseSchemaVersion < 16 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
-      }
+    if (databaseSchemaVersion < 14 && isBitcoin === true) {
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
+      await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+      await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
+      await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(14);
+    }
 
-      if (databaseSchemaVersion < 17 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
-      }
+    if (databaseSchemaVersion < 16 && isBitcoin === true) {
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
+      await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
+      await this.updateToSchemaVersion(16);
+    }
 
-      if (databaseSchemaVersion < 18 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
-      }
+    if (databaseSchemaVersion < 17 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
+      await this.updateToSchemaVersion(17);
+    }
 
-      if (databaseSchemaVersion < 19) {
-        await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
+    if (databaseSchemaVersion < 18 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
+      await this.updateToSchemaVersion(18);
+    }
+
+    if (databaseSchemaVersion < 19) {
+      await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
+      await this.updateToSchemaVersion(19);
+    }
+
+    if (databaseSchemaVersion < 20 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
+      await this.updateToSchemaVersion(20);
+    }
+
+    if (databaseSchemaVersion < 21) {
+      await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
+      await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
+      await this.updateToSchemaVersion(21);
+    }
+
+    if (databaseSchemaVersion < 22 && isBitcoin === true) {
+      await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
+      await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
+      await this.updateToSchemaVersion(22);
+    }
+
+    if (databaseSchemaVersion < 23) {
+      await this.$executeQuery('TRUNCATE `prices`');
+      await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
+      await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
+      await this.updateToSchemaVersion(23);
+    }
+
+    if (databaseSchemaVersion < 24 && isBitcoin == true) {
+      await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
+      await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
+      await this.updateToSchemaVersion(24);
+    }
+
+    if (databaseSchemaVersion < 25 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
+      await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
+      await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
+      await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
+      await this.updateToSchemaVersion(25);
+    }
+
+    if (databaseSchemaVersion < 26 && isBitcoin === true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated.`);
       }
-    } catch (e) {
-      throw e;
+      await this.$executeQuery(`TRUNCATE lightning_stats`);
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(26);
+    }
+
+    if (databaseSchemaVersion < 27 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(27);
+    }
+    
+    if (databaseSchemaVersion < 28 && isBitcoin === true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
+      }
+      await this.$executeQuery(`TRUNCATE lightning_stats`);
+      await this.$executeQuery(`TRUNCATE node_stats`);
+      await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
+      await this.updateToSchemaVersion(28);
+    }
+
+    if (databaseSchemaVersion < 29 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names'));
+      await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
+      await this.updateToSchemaVersion(29);
+    }
+
+    if (databaseSchemaVersion < 30 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
+      await this.updateToSchemaVersion(30);
+    }
+
+    if (databaseSchemaVersion < 31 && isBitcoin == true) { // Link blocks to prices
+      await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
+      await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
+      await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
+      await this.updateToSchemaVersion(31);
+    }
+
+    if (databaseSchemaVersion < 32 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(32);
+    }
+
+    if (databaseSchemaVersion < 33 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
+      await this.updateToSchemaVersion(33);
+    }
+
+    if (databaseSchemaVersion < 34 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(34);
+    }
+
+    if (databaseSchemaVersion < 35 && isBitcoin == true) {
+      await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
+      await this.updateToSchemaVersion(35);
+    }
+
+    if (databaseSchemaVersion < 36 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
+      await this.updateToSchemaVersion(36);
+    }
+
+    if (databaseSchemaVersion < 37 && isBitcoin == true) {
+      await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+      await this.updateToSchemaVersion(37);
+    }
+
+    if (databaseSchemaVersion < 38 && isBitcoin == true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
+      }
+      await this.$executeQuery(`TRUNCATE lightning_stats`);
+      await this.$executeQuery(`TRUNCATE node_stats`);
+      await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
+      await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
+      await this.updateToSchemaVersion(38);
+    }
+
+    if (databaseSchemaVersion < 39 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
+      await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
+      await this.updateToSchemaVersion(39);
+    }
+
+    if (databaseSchemaVersion < 40 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
+      await this.updateToSchemaVersion(40);
+    }
+
+    if (databaseSchemaVersion < 41 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
+      await this.updateToSchemaVersion(41);
+    }
+
+    if (databaseSchemaVersion < 42 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
+      await this.updateToSchemaVersion(42);
+    }
+
+    if (databaseSchemaVersion < 43 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+      await this.updateToSchemaVersion(43);
+    }
+
+    if (databaseSchemaVersion < 44 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE blocks_summaries SET template = NULL');
+      await this.updateToSchemaVersion(44);
+    }
+
+    if (databaseSchemaVersion < 45 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(45);
+    }
+
+    if (databaseSchemaVersion < 46) {
+      await this.$executeQuery(`ALTER TABLE blocks MODIFY blockTimestamp timestamp NOT NULL DEFAULT 0`);
+      await this.updateToSchemaVersion(46);
+    }
+
+    if (databaseSchemaVersion < 47) {
+      await this.$executeQuery('ALTER TABLE `blocks` ADD cpfp_indexed tinyint(1) DEFAULT 0');
+      await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
+      await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
+      await this.updateToSchemaVersion(47);
+    }
+
+    if (databaseSchemaVersion < 48 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node1_funding_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node2_funding_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node1_closing_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node2_closing_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD funding_ratio float unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `channels` ADD closed_by varchar(66) DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `channels` ADD single_funded tinyint(1) DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD outputs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(48);
+    }
+
+    if (databaseSchemaVersion < 49 && isBitcoin === true) {
+      await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
+      await this.updateToSchemaVersion(49);
     }
   }
 
@@ -228,7 +479,7 @@ class DatabaseMigration {
   /**
    * Small query execution wrapper to log all executed queries
    */
-  private async $executeQuery(query: string, silent: boolean = false): Promise<any> {
+  private async $executeQuery(query: string, silent = false): Promise<any> {
     if (!silent) {
       logger.debug('MIGRATIONS: Execute query:\n' + query);
     }
@@ -257,21 +508,17 @@ class DatabaseMigration {
    * Create the `state` table
    */
   private async $createMigrationStateTable(): Promise<void> {
-    try {
-      const query = `CREATE TABLE IF NOT EXISTS state (
-        name varchar(25) NOT NULL,
-        number int(11) NULL,
-        string varchar(100) NULL,
-        CONSTRAINT name_unique UNIQUE (name)
-      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
-      await this.$executeQuery(query);
+    const query = `CREATE TABLE IF NOT EXISTS state (
+      name varchar(25) NOT NULL,
+      number int(11) NULL,
+      string varchar(100) NULL,
+      CONSTRAINT name_unique UNIQUE (name)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+    await this.$executeQuery(query);
 
-      // Set initial values
-      await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
-      await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
-    } catch (e) {
-      throw e;
-    }
+    // Set initial values
+    await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
+    await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
   }
 
   /**
@@ -282,6 +529,8 @@ class DatabaseMigration {
     for (const query of this.getMigrationQueriesFromVersion(version)) {
       transactionQueries.push(query);
     }
+
+    logger.notice(`MIGRATIONS: ${version > 0 ? 'Upgrading' : 'Initializing'} database schema version number to ${DatabaseMigration.currentVersion}`);
     transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
 
     try {
@@ -305,6 +554,9 @@ class DatabaseMigration {
 
     if (version < 1) {
       if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
+        if (version > 0) {
+          logger.notice(`MIGRATIONS: Migrating (shifting) statistics table data`);
+        }
         queries.push(this.getShiftStatisticsQuery());
       }
     }
@@ -327,6 +579,10 @@ class DatabaseMigration {
     return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
   }
 
+  private async updateToSchemaVersion(version): Promise<void> {
+    await this.$executeQuery(`UPDATE state SET number = ${version} WHERE name = 'schema_version';`);
+  }
+
   /**
    * Print current database version
    */
@@ -470,7 +726,7 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  private getCreateRatesTableQuery(): string {
+  private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table
     return `CREATE TABLE IF NOT EXISTS rates (
       height int(10) unsigned NOT NULL,
       bisq_rates JSON NOT NULL,
@@ -478,8 +734,187 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateBlocksSummariesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_summaries (
+      height int(10) unsigned NOT NULL,
+      id varchar(65) NOT NULL,
+      transactions JSON NOT NULL,
+      PRIMARY KEY (id),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreatePricesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS prices (
+      time timestamp NOT NULL,
+      avg_prices JSON NOT NULL,
+      PRIMARY KEY (time)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateDifficultyAdjustmentsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS difficulty_adjustments (
+      time timestamp NOT NULL,
+      height int(10) unsigned NOT NULL,
+      difficulty double unsigned NOT NULL,
+      adjustment float NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (time)
+      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateLightningStatisticsQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS lightning_stats (
+      id int(11) NOT NULL AUTO_INCREMENT,
+      added datetime NOT NULL,
+      channel_count int(11) NOT NULL,
+      node_count int(11) NOT NULL,
+      total_capacity double unsigned NOT NULL,
+      PRIMARY KEY (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateNodesQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes (
+      public_key varchar(66) NOT NULL,
+      first_seen datetime NOT NULL,
+      updated_at datetime NOT NULL,
+      alias varchar(200) CHARACTER SET utf8mb4 NOT NULL,
+      color varchar(200) NOT NULL,
+      sockets text DEFAULT NULL,
+      PRIMARY KEY (public_key),
+      KEY alias (alias(10))
+      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateChannelsQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS channels (
+      id bigint(11) unsigned NOT NULL,
+      short_id varchar(15) NOT NULL DEFAULT '',
+      capacity bigint(20) unsigned NOT NULL,
+      transaction_id varchar(64) NOT NULL,
+      transaction_vout int(11) NOT NULL,
+      updated_at datetime DEFAULT NULL,
+      created datetime DEFAULT NULL,
+      status int(11) NOT NULL DEFAULT 0,
+      closing_transaction_id varchar(64) DEFAULT NULL,
+      closing_date datetime DEFAULT NULL,
+      closing_reason int(11) DEFAULT NULL,
+      node1_public_key varchar(66) NOT NULL,
+      node1_base_fee_mtokens bigint(20) unsigned DEFAULT NULL,
+      node1_cltv_delta int(11) DEFAULT NULL,
+      node1_fee_rate bigint(11) DEFAULT NULL,
+      node1_is_disabled tinyint(1) DEFAULT NULL,
+      node1_max_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
+      node1_min_htlc_mtokens bigint(20) DEFAULT NULL,
+      node1_updated_at datetime DEFAULT NULL,
+      node2_public_key varchar(66) NOT NULL,
+      node2_base_fee_mtokens bigint(20) unsigned DEFAULT NULL,
+      node2_cltv_delta int(11) DEFAULT NULL,
+      node2_fee_rate bigint(11) DEFAULT NULL,
+      node2_is_disabled tinyint(1) DEFAULT NULL,
+      node2_max_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
+      node2_min_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
+      node2_updated_at datetime DEFAULT NULL,
+      PRIMARY KEY (id),
+      KEY node1_public_key (node1_public_key),
+      KEY node2_public_key (node2_public_key),
+      KEY status (status),
+      KEY short_id (short_id),
+      KEY transaction_id (transaction_id),
+      KEY closing_transaction_id (closing_transaction_id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateNodesStatsQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS node_stats (
+      id int(11) unsigned NOT NULL AUTO_INCREMENT,
+      public_key varchar(66) NOT NULL DEFAULT '',
+      added date NOT NULL,
+      capacity bigint(20) unsigned NOT NULL DEFAULT 0,
+      channels int(11) unsigned NOT NULL DEFAULT 0,
+      PRIMARY KEY (id),
+      UNIQUE KEY added (added,public_key),
+      KEY public_key (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateBlocksAuditsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_audits (
+      time timestamp NOT NULL,
+      hash varchar(65) NOT NULL,
+      height int(10) unsigned NOT NULL,
+      missing_txs JSON NOT NULL,
+      added_txs JSON NOT NULL,
+      match_rate float unsigned NOT NULL,
+      PRIMARY KEY (hash),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateGeoNamesTableQuery(): string {
+    return `CREATE TABLE geo_names (
+      id int(11) unsigned NOT NULL,
+      type enum('city','country','division','continent') NOT NULL,
+      names text DEFAULT NULL,
+      UNIQUE KEY id (id,type),
+      KEY id_2 (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateBlocksPricesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_prices (
+      height int(10) unsigned NOT NULL,
+      price_id int(10) unsigned NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (price_id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateLNNodesSocketsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_sockets (
+      public_key varchar(66) NOT NULL,
+      socket varchar(100) NOT NULL,
+      type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL,
+      UNIQUE KEY public_key_socket (public_key, socket),
+      INDEX (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateLNNodeRecordsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_records (
+      public_key varchar(66) NOT NULL,
+      type int(10) unsigned NOT NULL,
+      payload blob NOT NULL,
+      UNIQUE KEY public_key_type (public_key, type),
+      INDEX (public_key),
+      FOREIGN KEY (public_key)
+        REFERENCES nodes (public_key)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCPFPTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS cpfp_clusters (
+      root varchar(65) NOT NULL,
+      height int(10) NOT NULL,
+      txs JSON DEFAULT NULL,
+      fee_rate double unsigned NOT NULL,
+      PRIMARY KEY (root)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateTransactionsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS transactions (
+      txid varchar(65) NOT NULL,
+      cluster varchar(65) DEFAULT NULL,
+      PRIMARY KEY (txid),
+      FOREIGN KEY (cluster) REFERENCES cpfp_clusters (root) ON DELETE SET NULL
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates'];
+    const allowedTables = ['blocks', 'hashrates', 'prices'];
 
     try {
       for (const table of tables) {

backend/src/api/difficulty-adjustment.ts

@@ -2,65 +2,100 @@ import config from '../config';
 import { IDifficultyAdjustment } from '../mempool.interfaces';
 import blocks from './blocks';
 
-class DifficultyAdjustmentApi {
-  constructor() { }
+export interface DifficultyAdjustment {
+  progressPercent: number;       // Percent: 0 to 100
+  difficultyChange: number;      // Percent: -75 to 300
+  estimatedRetargetDate: number; // Unix time in ms
+  remainingBlocks: number;       // Block count
+  remainingTime: number;         // Duration of time in ms
+  previousRetarget: number;      // Percent: -75 to 300
+  nextRetargetHeight: number;    // Block Height
+  timeAvg: number;               // Duration of time in ms
+  timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
+}
 
-  public getDifficultyAdjustment(): IDifficultyAdjustment {
+export function calcDifficultyAdjustment(
+  DATime: number,
+  nowSeconds: number,
+  blockHeight: number,
+  previousRetarget: number,
+  network: string,
+  latestBlockTimestamp: number,
+): DifficultyAdjustment {
+  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
+  const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
+  const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
+  const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
+
+  const diffSeconds = nowSeconds - DATime;
+  const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
+  const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
+  const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
+  const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
+
+  let difficultyChange = 0;
+  let timeAvgSecs = BLOCK_SECONDS_TARGET;
+  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+    timeAvgSecs = diffSeconds / blocksInEpoch;
+    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+    // Max increase is x4 (+300%)
+    if (difficultyChange > 300) {
+      difficultyChange = 300;
+    }
+    // Max decrease is /4 (-75%)
+    if (difficultyChange < -75) {
+      difficultyChange = -75;
+    }
+  }
+
+  // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+  // therefore the time between blocks will always be below 20 minutes (1200s).
+  let timeOffset = 0;
+  if (network === 'testnet') {
+    if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
+    }
+
+    const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
+    if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
+    }
+  }
+
+  const timeAvg = Math.floor(timeAvgSecs * 1000);
+  const remainingTime = remainingBlocks * timeAvg;
+  const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
+
+  return {
+    progressPercent,
+    difficultyChange,
+    estimatedRetargetDate,
+    remainingBlocks,
+    remainingTime,
+    previousRetarget,
+    nextRetargetHeight,
+    timeAvg,
+    timeOffset,
+  };
+}
+
+class DifficultyAdjustmentApi {
+  public getDifficultyAdjustment(): IDifficultyAdjustment | null {
     const DATime = blocks.getLastDifficultyAdjustmentTime();
     const previousRetarget = blocks.getPreviousDifficultyRetarget();
     const blockHeight = blocks.getCurrentBlockHeight();
     const blocksCache = blocks.getBlocks();
     const latestBlock = blocksCache[blocksCache.length - 1];
-
-    const now = new Date().getTime() / 1000;
-    const diff = now - DATime;
-    const blocksInEpoch = blockHeight % 2016;
-    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
-    const remainingBlocks = 2016 - blocksInEpoch;
-    const nextRetargetHeight = blockHeight + remainingBlocks;
-
-    let difficultyChange = 0;
-    if (remainingBlocks < 1870) {
-      if (blocksInEpoch > 0) {
-        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
-      }
-      if (difficultyChange > 300) {
-        difficultyChange = 300;
-      }
-      if (difficultyChange < -75) {
-        difficultyChange = -75;
-      }
+    if (!latestBlock) {
+      return null;
     }
+    const nowSeconds = Math.floor(new Date().getTime() / 1000);
 
-    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
-
-    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
-    // therefore the time between blocks will always be below 20 minutes (1200s).
-    let timeOffset = 0;
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      if (timeAvgMins > 20) {
-        timeAvgMins = 20;
-      }
-      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
-        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
-      }
-    }
-
-    const timeAvg = timeAvgMins * 60 * 1000 ;
-    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
-    const estimatedRetargetDate = remainingTime + now;
-
-    return {
-      progressPercent,
-      difficultyChange,
-      estimatedRetargetDate,
-      remainingBlocks,
-      remainingTime,
-      previousRetarget,
-      nextRetargetHeight,
-      timeAvg,
-      timeOffset,
-    };
+    return calcDifficultyAdjustment(
+      DATime, nowSeconds, blockHeight, previousRetarget,
+      config.MEMPOOL.NETWORK, latestBlock.timestamp
+    );
   }
 }
 

backend/src/api/disk-cache.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 const fsPromises = fs.promises;
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
@@ -19,7 +19,7 @@ class DiskCache {
   constructor() { }
 
   async $saveCacheToDisk(): Promise<void> {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
       return;
     }
     if (this.isWritingCache) {
@@ -43,14 +43,15 @@ class DiskCache {
       await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
         cacheSchemaVersion: this.cacheSchemaVersion,
         blocks: blocks.getBlocks(),
+        blockSummaries: blocks.getBlockSummaries(),
         mempool: {},
         mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), {flag: 'w'});
+      }), { flag: 'w' });
       for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
         await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), {flag: 'w'});
+        }), { flag: 'w' });
       }
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
@@ -66,7 +67,7 @@ class DiskCache {
       fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
     }
   }
-  
+
   loadMempoolCache() {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
@@ -109,6 +110,7 @@ class DiskCache {
 
       memPool.setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
+      blocks.setBlockSummaries(data.blockSummaries || []);
     } catch (e) {
       logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }

backend/src/api/explorer/channels.api.ts

@@ -0,0 +1,700 @@
+import logger from '../../logger';
+import DB from '../../database';
+import nodesApi from './nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { Common } from '../common';
+
+class ChannelsApi {
+  public async $getAllChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getAllChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
+    try {
+      let select: string;
+      if (style === 'widget') {
+        select = `
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      } else {
+        select = `
+          nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      }
+
+      const params: string[] = [];
+      let query = `SELECT ${select}
+        FROM channels
+        JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
+        JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
+        WHERE channels.status = 1
+          AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
+          AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
+      `;
+
+      if (publicKey !== undefined) {
+        query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
+        params.push(publicKey);
+        params.push(publicKey);
+      } else {
+        query += ` AND channels.capacity > 1000000
+          GROUP BY nodes_1.public_key, nodes_2.public_key
+          ORDER BY channels.capacity DESC
+          LIMIT 10000
+        `;        
+      }
+
+      const [rows]: any = await DB.query(query, params);
+      return rows.map((row) => {
+        if (style === 'widget') {
+          return [
+            row.node1_longitude, row.node1_latitude,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        } else {
+          return [
+            row.node1_public_key, row.node1_alias,
+            row.node1_longitude, row.node1_latitude,
+            row.node2_public_key, row.node2_alias,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        }
+      });
+    } catch (e) {
+      logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $searchChannelsById(search: string): Promise<any[]> {
+    try {
+      const searchStripped = search.replace('%', '') + '%';
+      const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
+      const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
+      return rows;
+    } catch (e) {
+      logger.err('$searchChannelsById error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsByStatus(status: number | number[]): Promise<any[]> {
+    try {
+      let query: string;
+      if (Array.isArray(status)) {
+        query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`;
+      } else {
+        query = `SELECT * FROM channels WHERE status = ?`;
+      }
+      const [rows]: any = await DB.query(query, [status]);
+      return rows;
+    } catch (e) {
+      logger.err('$getChannelsByStatus error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getClosedChannelsWithoutReason(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason IS NULL AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getClosedChannelsWithoutReason error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getUnresolvedClosedChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
+    try {
+      const query = `
+        SELECT channels.*
+        FROM channels
+        WHERE channels.source_checked != 1
+      `;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE created IS NULL`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getChannelsWithoutCreatedDate error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannel(id: string): Promise<any> {
+    try {
+      const query = `
+        SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
+          n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
+          channels.*,
+          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
+        LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
+        WHERE (
+          ns1.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node1_public_key
+          )
+          AND ns2.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node2_public_key
+          )
+        )
+        AND channels.id = ?
+      `;
+
+      const [rows]: any = await DB.query(query, [id]);
+      if (rows[0]) {
+        return this.convertChannel(rows[0]);
+      }
+    } catch (e) {
+      logger.err('$getChannel error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsStats(): Promise<any> {
+    try {
+      // Feedback from zerofeerouting:
+      // "I would argue > 5000ppm can be ignored. Channels charging more than .5% fee are ignored by CLN for example."
+      const ignoredFeeRateThreshold = 5000;
+      const ignoredBaseFeeThreshold = 5000;
+
+      // Capacity
+      let query = `SELECT AVG(capacity) AS avgCapacity FROM channels WHERE status = 1 ORDER BY capacity`;
+      const [avgCapacity]: any = await DB.query(query);
+
+      query = `SELECT capacity FROM channels WHERE status = 1 ORDER BY capacity`;
+      let [capacity]: any = await DB.query(query);
+      capacity = capacity.map(capacity => capacity.capacity);
+      const medianCapacity = capacity[Math.floor(capacity.length / 2)];
+
+      // Fee rates
+      query = `SELECT node1_fee_rate FROM channels WHERE node1_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`;
+      let [feeRates1]: any = await DB.query(query);
+      feeRates1 = feeRates1.map(rate => rate.node1_fee_rate);
+      query = `SELECT node2_fee_rate FROM channels WHERE node2_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`;
+      let [feeRates2]: any = await DB.query(query);
+      feeRates2 = feeRates2.map(rate => rate.node2_fee_rate);
+
+      let feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b);
+      let avgFeeRate = 0;
+      for (const rate of feeRates) {
+        avgFeeRate += rate; 
+      }
+      avgFeeRate /= feeRates.length;
+      const medianFeeRate = feeRates[Math.floor(feeRates.length / 2)];
+
+      // Base fees
+      query = `SELECT node1_base_fee_mtokens FROM channels WHERE node1_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`;
+      let [baseFees1]: any = await DB.query(query);
+      baseFees1 = baseFees1.map(rate => rate.node1_base_fee_mtokens);
+      query = `SELECT node2_base_fee_mtokens FROM channels WHERE node2_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`;
+      let [baseFees2]: any = await DB.query(query);
+      baseFees2 = baseFees2.map(rate => rate.node2_base_fee_mtokens);
+
+      let baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b);
+      let avgBaseFee = 0;
+      for (const fee of baseFees) {
+        avgBaseFee += fee; 
+      }
+      avgBaseFee /= baseFees.length;
+      const medianBaseFee = feeRates[Math.floor(baseFees.length / 2)];
+      
+      return {
+        avgCapacity: parseInt(avgCapacity[0].avgCapacity, 10),
+        avgFeeRate: avgFeeRate,
+        avgBaseFee: avgBaseFee,
+        medianCapacity: medianCapacity,
+        medianFeeRate: medianFeeRate,
+        medianBaseFee: medianBaseFee,
+      }
+
+    } catch (e) {
+      logger.err(`Cannot calculate channels statistics. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
+    try {
+      const query = `
+        SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
+      `;
+      const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
+      const channels = rows.map((row) => this.convertChannel(row));
+      return channels;
+    } catch (e) {
+      logger.err('$getChannelByTransactionId error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelByClosingId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.closing_transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        rows[0].outputs = JSON.parse(rows[0].outputs);
+        return rows[0];
+      }
+    } catch (e) {
+      logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsByOpeningId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        return rows.map(row => {
+          row.outputs = JSON.parse(row.outputs);
+          return row;
+        });
+      }
+    } catch (e) {
+      logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_closing_balance = ?,
+          node2_closing_balance = ?,
+          closed_by = ?,
+          closing_fee = ?,
+          outputs = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_closing_balance || 0,
+        channelInfo.node2_closing_balance || 0,
+        channelInfo.closed_by,
+        channelInfo.closing_fee || 0,
+        JSON.stringify(channelInfo.outputs),
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_funding_balance = ?,
+          node2_funding_balance = ?,
+          funding_ratio = ?,
+          single_funded = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_funding_balance || 0,
+        channelInfo.node2_funding_balance || 0,
+        channelInfo.funding_ratio,
+        channelInfo.single_funded ? 1 : 0,
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $markChannelSourceChecked(id: string): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels
+        SET source_checked = 1
+        WHERE id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [id]);
+    } catch (e) {
+      logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
+    try {
+      let channelStatusFilter;
+      if (status === 'open') {
+        channelStatusFilter = '< 2';
+      } else if (status === 'active') {
+        channelStatusFilter = '= 1';
+      } else if (status === 'closed') {
+        channelStatusFilter = '= 2';
+      } else {
+        throw new Error('getChannelsForNode: Invalid status requested');
+      }
+
+      // Channels originating from node
+      let query = `
+        SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
+          channels.status, channels.node1_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
+        WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsFromNode]: any = await DB.query(query, [public_key]);
+
+      // Channels incoming to node
+      query = `
+        SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
+          channels.status, channels.node2_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
+        WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsToNode]: any = await DB.query(query, [public_key]);
+
+      let allChannels = channelsFromNode.concat(channelsToNode);
+      allChannels.sort((a, b) => {
+        if (status === 'closed') {
+          if (!b.closing_date && !a.closing_date) {
+            return (b.updated_at ?? 0) - (a.updated_at ?? 0);
+          } else {
+            return (b.closing_date ?? 0) - (a.closing_date ?? 0);
+          }
+        } else {
+          return b.capacity - a.capacity;
+        }
+      });
+
+      if (index >= 0) {
+        allChannels = allChannels.slice(index, index + length);
+      } else if (index === -1) { // Node channels tree chart
+        allChannels = allChannels.slice(0, 1000);
+      }
+
+      const channels: any[] = []
+      for (const row of allChannels) {
+        let channel;
+        if (index >= 0) {
+          const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
+          channel = {
+            status: row.status,
+            closing_reason: row.closing_reason,
+            closing_date: row.closing_date,
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+              channels: activeChannelsStats.active_channel_count ?? 0,
+              capacity: activeChannelsStats.capacity ?? 0,
+            }
+          };
+        } else if (index === -1) {
+          channel = {
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+            }
+          };
+        }
+
+        channels.push(channel);
+      }
+
+      return channels;
+    } catch (e) {
+      logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsCountForNode(public_key: string, status: string): Promise<any> {
+    try {
+      // Default active and inactive channels
+      let statusQuery = '< 2';
+      // Closed channels only
+      if (status === 'closed') {
+        statusQuery = '= 2';
+      }
+      const query = `
+        SELECT COUNT(*) AS count
+        FROM channels
+        WHERE (node1_public_key = ? OR node2_public_key = ?)
+        AND status ${statusQuery}
+      `;
+      const [rows]: any = await DB.query(query, [public_key, public_key]);
+      return rows[0]['count'];
+    } catch (e) {
+      logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private convertChannel(channel: any): any {
+    return {
+      'id': channel.id,
+      'short_id': channel.short_id,
+      'capacity': channel.capacity,
+      'transaction_id': channel.transaction_id,
+      'transaction_vout': channel.transaction_vout,
+      'closing_transaction_id': channel.closing_transaction_id,
+      'closing_fee': channel.closing_fee,
+      'closing_reason': channel.closing_reason,
+      'closing_date': channel.closing_date,
+      'updated_at': channel.updated_at,
+      'created': channel.created,
+      'status': channel.status,
+      'funding_ratio': channel.funding_ratio,
+      'closed_by': channel.closed_by,
+      'single_funded': !!channel.single_funded,
+      'node_left': {
+        'alias': channel.alias_left,
+        'public_key': channel.node1_public_key,
+        'channels': channel.channels_left,
+        'capacity': channel.capacity_left,
+        'base_fee_mtokens': channel.node1_base_fee_mtokens,
+        'cltv_delta': channel.node1_cltv_delta,
+        'fee_rate': channel.node1_fee_rate,
+        'is_disabled': channel.node1_is_disabled,
+        'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
+        'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
+        'updated_at': channel.node1_updated_at,
+        'longitude': channel.node1_longitude,
+        'latitude': channel.node1_latitude,
+        'funding_balance': channel.node1_funding_balance,
+        'closing_balance': channel.node1_closing_balance,
+        'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
+      },
+      'node_right': {
+        'alias': channel.alias_right,
+        'public_key': channel.node2_public_key,
+        'channels': channel.channels_right,
+        'capacity': channel.capacity_right,
+        'base_fee_mtokens': channel.node2_base_fee_mtokens,
+        'cltv_delta': channel.node2_cltv_delta,
+        'fee_rate': channel.node2_fee_rate,
+        'is_disabled': channel.node2_is_disabled,
+        'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
+        'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
+        'updated_at': channel.node2_updated_at,
+        'longitude': channel.node2_longitude,
+        'latitude': channel.node2_latitude,
+        'funding_balance': channel.node2_funding_balance,
+        'closing_balance': channel.node2_closing_balance,
+        'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
+      },
+    };
+  }
+
+  /**
+   * Save or update a channel present in the graph
+   */
+  public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
+    const [ txid, vout ] = channel.chan_point.split(':');
+
+    const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
+    const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
+
+    const query = `INSERT INTO channels
+      (
+        id,
+        short_id,
+        capacity,
+        transaction_id,
+        transaction_vout,
+        updated_at,
+        status,
+        node1_public_key,
+        node1_base_fee_mtokens,
+        node1_cltv_delta,
+        node1_fee_rate,
+        node1_is_disabled,
+        node1_max_htlc_mtokens,
+        node1_min_htlc_mtokens,
+        node1_updated_at,
+        node2_public_key,
+        node2_base_fee_mtokens,
+        node2_cltv_delta,
+        node2_fee_rate,
+        node2_is_disabled,
+        node2_max_htlc_mtokens,
+        node2_min_htlc_mtokens,
+        node2_updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+        capacity = ?,
+        updated_at = ?,
+        status = ${status},
+        node1_public_key = ?,
+        node1_base_fee_mtokens = ?,
+        node1_cltv_delta = ?,
+        node1_fee_rate = ?,
+        node1_is_disabled = ?,
+        node1_max_htlc_mtokens = ?,
+        node1_min_htlc_mtokens = ?,
+        node1_updated_at = ?,
+        node2_public_key = ?,
+        node2_base_fee_mtokens = ?,
+        node2_cltv_delta = ?,
+        node2_fee_rate = ?,
+        node2_is_disabled = ?,
+        node2_max_htlc_mtokens = ?,
+        node2_min_htlc_mtokens = ?,
+        node2_updated_at = ?
+      ;`;
+
+    await DB.query(query, [
+      Common.channelShortIdToIntegerId(channel.channel_id),
+      Common.channelIntegerIdToShortId(channel.channel_id),
+      channel.capacity,
+      txid,
+      vout,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update),
+      channel.capacity,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update)
+    ]);
+  }
+
+  /**
+   * Set all channels not in `graphChannelsIds` as inactive (status = 0)
+   */
+  public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
+    if (graphChannelsIds.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE id NOT IN (
+          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
+        )
+        AND status != 2
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
+    try {
+      const query = `
+        SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
+        FROM channels
+        WHERE node1_public_key = ?
+      `;
+      const [rows]: any[] = await DB.query(query, [publicKey]);
+      if (rows.length > 0) {
+        return rows[0].updated_at;
+      }
+    } catch (e) {
+      logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
+    }
+    return 0;
+  }
+}
+
+export default new ChannelsApi();

backend/src/api/explorer/channels.routes.ts

@@ -0,0 +1,126 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import channelsApi from './channels.api';
+
+class ChannelsRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/txids', this.$getChannelsByTransactionIds)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getAllChannelsGeo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo/:publicKey', this.$getAllChannelsGeo)
+    ;
+  }
+
+  private async $searchChannelsById(req: Request, res: Response) {
+    try {
+      const channels = await channelsApi.$searchChannelsById(req.params.search);
+      res.json(channels);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getChannel(req: Request, res: Response) {
+    try {
+      const channel = await channelsApi.$getChannel(req.params.short_id);
+      if (!channel) {
+        res.status(404).send('Channel not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channel);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getChannelsForNode(req: Request, res: Response) {
+    try {
+      if (typeof req.query.public_key !== 'string') {
+        res.status(400).send('Missing parameter: public_key');
+        return;
+      }
+
+      const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
+      const status: string = typeof req.query.status === 'string' ? req.query.status : '';
+
+      if (index < -1) {
+        res.status(400).send('Invalid index');
+      }
+      if (['open', 'active', 'closed'].includes(status) === false) {
+        res.status(400).send('Invalid status');
+      }
+
+      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
+      const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.header('X-Total-Count', channelsCount.toString());
+      res.json(channels);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
+    try {
+      if (!Array.isArray(req.query.txId)) {
+        res.status(400).send('Not an array');
+        return;
+      }
+      const txIds: string[] = [];
+      for (const _txId in req.query.txId) {
+        if (typeof req.query.txId[_txId] === 'string') {
+          txIds.push(req.query.txId[_txId].toString());
+        }
+      }
+      const channels = await channelsApi.$getChannelsByTransactionId(txIds);
+      const result: any[] = [];
+      for (const txid of txIds) {
+        const inputs: any = {};
+        const outputs: any = {};
+        // Assuming that we only have one lightning close input in each transaction. This may not be true in the future
+        const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
+        if (foundChannelsFromInput) {
+          inputs[0] = foundChannelsFromInput;
+        }
+        const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
+        for (const output of foundChannelsFromOutputs) {
+          outputs[output.transaction_vout] = output;
+        }
+        result.push({
+          inputs,
+          outputs,
+        });
+      }
+
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getAllChannelsGeo(req: Request, res: Response) {
+    try {
+      const style: string = typeof req.query.style === 'string' ? req.query.style : '';
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channels);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+}
+
+export default new ChannelsRoutes();

backend/src/api/explorer/general.routes.ts

@@ -0,0 +1,58 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import nodesApi from './nodes.api';
+import channelsApi from './channels.api';
+import statisticsApi from './statistics.api';
+class GeneralLightningRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/search', this.$searchNodesAndChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics/latest', this.$getGeneralStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics/:interval', this.$getStatistics)
+    ;
+  }
+
+  private async $searchNodesAndChannels(req: Request, res: Response) {
+    if (typeof req.query.searchText !== 'string') {
+      res.status(400).send('Missing parameter: searchText');
+      return;
+    }
+    try {
+      const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.query.searchText);
+      const channels = await channelsApi.$searchChannelsById(req.query.searchText);
+      res.json({
+        nodes: nodes,
+        channels: channels,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getStatistics(req: Request, res: Response) {
+    try {
+      const statistics = await statisticsApi.$getStatistics(req.params.interval);
+      const statisticsCount = await statisticsApi.$getStatisticsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', statisticsCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getGeneralStats(req: Request, res: Response) {
+    try {
+      const statistics = await statisticsApi.$getLatestStatistics();
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new GeneralLightningRoutes();

backend/src/api/explorer/nodes.api.ts

@@ -0,0 +1,702 @@
+import logger from '../../logger';
+import DB from '../../database';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
+
+class NodesApi {
+  public async $getWorldNodes(): Promise<any> {
+    try {
+      let query = `
+        SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+        CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+        CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+        nodes.longitude, nodes.latitude,
+        geo_names_country.names as country, geo_names_iso.names as isoCode
+        FROM nodes
+        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+        ORDER BY capacity
+      `;
+
+      const [nodes]: any[] = await DB.query(query);
+
+      for (let i = 0; i < nodes.length; ++i) {
+        nodes[i].country = JSON.parse(nodes[i].country);
+      }
+
+      query = `
+        SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
+        FROM nodes
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+      `;
+
+      const [maximums]: any[] = await DB.query(query);
+      
+      return {
+        maxLiquidity: maximums[0].maxLiquidity,
+        maxChannels: maximums[0].maxChannels,
+        nodes: nodes.map(node => [
+          node.longitude, node.latitude,
+          node.publicKey, node.alias, node.capacity, node.channels,
+          node.country, node.isoCode
+        ])
+      };
+    } catch (e) {
+      logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public async $getNode(public_key: string): Promise<any> {
+    try {
+      // General info
+      let query = `
+        SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
+          UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
+          as_number, city_id, country_id, subdivision_id, longitude, latitude,
+          geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
+          geo_names_country.names as country, geo_names_subdivision.names as subdivision
+        FROM nodes
+        LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
+        LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = subdivision_id
+        LEFT JOIN geo_names geo_names_country on geo_names_country.id = country_id
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE public_key = ?
+      `;
+      let [rows]: any[] = await DB.query(query, [public_key]);
+      if (rows.length === 0) {
+        throw new Error(`This node does not exist, or our node is not seeing it yet`);
+      }
+
+      const node = rows[0];
+      node.as_organization = JSON.parse(node.as_organization);
+      node.subdivision = JSON.parse(node.subdivision);
+      node.city = JSON.parse(node.city);
+      node.country = JSON.parse(node.country);
+
+      // Active channels and capacity
+      const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
+      node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
+      node.capacity = activeChannelsStats.capacity ?? 0;
+
+      // Opened channels count
+      query = `
+        SELECT count(short_id) as opened_channel_count
+        FROM channels
+        WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.opened_channel_count = 0;
+      if (rows.length > 0) {
+        node.opened_channel_count = rows[0].opened_channel_count;
+      }
+
+      // Closed channels count
+      query = `
+        SELECT count(short_id) as closed_channel_count
+        FROM channels
+        WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.closed_channel_count = 0;
+      if (rows.length > 0) {
+        node.closed_channel_count = rows[0].closed_channel_count;
+      }
+
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
+      return node;
+    } catch (e) {
+      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
+    const query = `
+      SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
+      FROM channels
+      WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+    `;
+    const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
+    if (rows.length > 0) {
+      return {
+        active_channel_count: rows[0].active_channel_count,
+        capacity: rows[0].capacity
+      };
+    } else {
+      return null;
+    }
+  }
+
+  public async $getFeeHistogram(node_public_key: string): Promise<unknown> {
+    try {
+      const inQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node2_fee_rate WHEN node2_public_key = ? THEN node1_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [inRows]: any[] = await DB.query(inQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      const outQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node1_fee_rate WHEN node2_public_key = ? THEN node2_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [outRows]: any[] = await DB.query(outQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      return {
+        incoming: inRows.length > 0 ? inRows : [],
+        outgoing: outRows.length > 0 ? outRows : [],
+      };
+    } catch (e) {
+      logger.err(`Cannot get node fee distribution for ${node_public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  public async $getAllNodes(): Promise<any> {
+    try {
+      const query = `SELECT * FROM nodes`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getAllNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNodeStats(public_key: string): Promise<any> {
+    try {
+      const query = `
+        SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
+        FROM node_stats
+        WHERE public_key = ?
+        ORDER BY added DESC
+      `;
+      const [rows]: any = await DB.query(query, [public_key]);
+      return rows;
+    } catch (e) {
+      logger.err('$getNodeStats error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTopCapacityNodes(full: boolean): Promise<ITopNodesPerCapacity[]> {
+    try {
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.capacity
+          FROM nodes
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTopChannelsNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels
+          FROM nodes
+          ORDER BY channels DESC
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
+      const latestDate = rows[0].maxAdded;
+
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            node_stats.channels
+          FROM node_stats
+          JOIN nodes ON nodes.public_key = node_stats.public_key
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
+            CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM node_stats
+          RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $searchNodeByPublicKeyOrAlias(search: string) {
+    try {
+      const publicKeySearch = search.replace('%', '') + '%';
+      const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
+      const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
+      return rows;
+    } catch (e) {
+      logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNodesISPRanking() {
+    try {
+      let query = '';
+
+      // List all channels and the two linked ISP
+      query = `
+        SELECT short_id, channels.capacity,
+          channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1
+        ORDER BY short_id DESC
+      `;
+      const [channelsIsp]: any = await DB.query(query);
+
+      // Sum channels capacity and node count per ISP
+      const ispList = {};
+      for (const channel of channelsIsp) {
+        const isp1 = JSON.parse(channel.isp1);
+        const isp2 = JSON.parse(channel.isp2);
+
+        if (!ispList[isp1]) {
+          ispList[isp1] = {
+            id: channel.isp1ID.toString(),
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp1].id.indexOf(channel.isp1ID) === -1) {
+          ispList[isp1].id += ',' + channel.isp1ID.toString();
+        }
+
+        if (!ispList[isp2]) {
+          ispList[isp2] = {
+            id: channel.isp2ID.toString(),
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp2].id.indexOf(channel.isp2ID) === -1) {
+          ispList[isp2].id += ',' + channel.isp2ID.toString();
+        }
+        
+        ispList[isp1].capacity += channel.capacity;
+        ispList[isp1].channels += 1;
+        ispList[isp1].nodes[channel.node1PublicKey] = true;
+        ispList[isp2].capacity += channel.capacity;
+        ispList[isp2].channels += 1;
+        ispList[isp2].nodes[channel.node2PublicKey] = true;
+      }
+
+      const ispRanking: any[] = [];
+      for (const isp of Object.keys(ispList)) {
+        ispRanking.push([
+          ispList[isp].id,
+          isp,
+          ispList[isp].capacity,
+          ispList[isp].channels,
+          Object.keys(ispList[isp].nodes).length,
+        ]);
+      }
+
+      // Total active channels capacity
+      query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`;
+      const [totalCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have at least one node on clearnet
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks LIKE '%ipv%'
+      `;
+      const [clearnetCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have both nodes on Tor 
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks NOT LIKE '%ipv%' AND
+          channels_tmp.networks NOT LIKE '%dns%' AND
+          channels_tmp.networks NOT LIKE '%websocket%'
+      `;
+      const [torCapacity]: any = await DB.query(query);
+
+      const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10);
+      const torCapacityValue = parseInt(torCapacity[0].capacity, 10);
+      const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue;
+
+      return {
+        clearnetCapacity: clearnetCapacityValue,
+        torCapacity: torCapacityValue,
+        unknownCapacity: unknownCapacityValue,
+        ispRanking: ispRanking,
+      };
+    } catch (e) {
+      logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerCountry(countryId: string) {
+    try {
+      const query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
+        WHERE geo_names_country.id = ?
+        ORDER BY capacity DESC
+      `;
+
+      const [rows]: any = await DB.query(query, [countryId]);
+      for (let i = 0; i < rows.length; ++i) {
+        rows[i].country = JSON.parse(rows[i].country);
+        rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows[i].isp = JSON.parse(rows[i].isp);
+      }
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get nodes for country id ${countryId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerISP(ISPId: string) {
+    try {
+      let query = `
+        SELECT channels.node1_public_key AS node1PublicKey, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1 AND (node1.as_number IN (?) OR node2.as_number IN (?))
+        ORDER BY short_id DESC
+      `;
+
+      const IPSIds = ISPId.split(',');
+      const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
+      if (!rows || rows.length === 0) {
+        return [];
+      }
+
+      const nodes = {};
+
+      const intISPIds: number[] = [];
+      for (const ispId of IPSIds) {
+        intISPIds.push(parseInt(ispId, 10));
+      }
+
+      for (const channel of rows) {
+        if (intISPIds.includes(channel.isp1ID)) {
+          nodes[channel.node1PublicKey] = true;
+        }
+        if (intISPIds.includes(channel.isp2ID)) {
+          nodes[channel.node2PublicKey] = true;
+        }
+      }
+
+      query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        WHERE nodes.public_key IN (?)
+        ORDER BY capacity DESC
+      `;
+
+      const [rows2]: any = await DB.query(query, [Object.keys(nodes)]);
+      for (let i = 0; i < rows2.length; ++i) {
+        rows2[i].country = JSON.parse(rows2[i].country);
+        rows2[i].city = JSON.parse(rows2[i].city);
+        rows2[i].subdivision = JSON.parse(rows2[i].subdivision);
+      }
+      return rows2;
+
+    } catch (e) {
+      logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesCountries() {
+    try {
+      let query = `SELECT geo_names.names as names, geo_names_iso.names as iso_code, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
+        FROM nodes
+        JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        GROUP BY country_id
+        ORDER BY COUNT(DISTINCT nodes.public_key) DESC
+      `;
+      const [nodesCountPerCountry]: any = await DB.query(query);
+
+      query = `SELECT COUNT(*) as total FROM nodes WHERE country_id IS NOT NULL`;
+      const [nodesWithAS]: any = await DB.query(query);
+
+      const nodesPerCountry: any[] = [];
+      for (const country of nodesCountPerCountry) {
+        nodesPerCountry.push({
+          name: JSON.parse(country.names),
+          iso: country.iso_code, 
+          count: country.nodesCount,
+          share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100,
+          capacity: country.capacity,
+        })
+      }
+
+      return nodesPerCountry;
+    } catch (e) {
+      logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  /**
+   * Save or update a node present in the graph
+   */
+  public async $saveNode(node: ILightningApi.Node): Promise<void> {
+    try {
+      const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
+      const query = `INSERT INTO nodes(
+          public_key,
+          first_seen,
+          updated_at,
+          alias,
+          alias_search,
+          color,
+          sockets,
+          status
+        )
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
+        ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
+
+      await DB.query(query, [
+        node.pub_key,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+      ]);
+    } catch (e) {
+      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update node sockets
+   */
+  public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
+    const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
+    try {
+      await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
+    } catch (e) {
+      logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /**
+   * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   */
+   public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
+    if (graphNodesPubkeys.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE nodes
+        SET status = 0
+        WHERE public_key NOT IN (
+          ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
+        )
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private aliasToSearchText(str: string): string {
+    return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '');
+  }
+}
+
+export default new NodesApi();

backend/src/api/explorer/nodes.routes.ts

@@ -0,0 +1,259 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import nodesApi from './nodes.api';
+import DB from '../../database';
+import { INodesRanking } from '../../mempool.interfaces';
+
+class NodesRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/liquidity', this.$getTopNodesByCapacity)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/fees/histogram', this.$getFeeHistogram)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
+    ;
+  }
+
+  private async $searchNode(req: Request, res: Response) {
+    try {
+      const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.params.search);
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodeGroup(req: Request, res: Response) {
+    try {
+      let nodesList;
+      let nodes: any[] = [];
+      switch (config.MEMPOOL.NETWORK) {
+        case 'testnet':
+          nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
+          break;
+        case 'signet':
+          nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
+          break;
+        default:
+          nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
+      }
+
+      for (let pubKey of nodesList) {
+        try {
+          const node = await nodesApi.$getNode(pubKey);
+          if (node) {
+            nodes.push(node);
+          }
+        } catch (e) {}
+      }
+      
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNode(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getNode(req.params.public_key);
+      if (!node) {
+        res.status(404).send('Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalNodeStats(req: Request, res: Response) {
+    try {
+      const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getFeeHistogram(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getFeeHistogram(req.params.public_key);
+      if (!node) {
+        res.status(404).send('Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);
+      const topChannelsNodes = await nodesApi.$getTopChannelsNodes(false);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(<INodesRanking>{
+        topByCapacity: topCapacityNodes,
+        topByChannels: topChannelsNodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodesByCapacity(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodesByChannels(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopChannelsNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getOldestNodes(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getOldestNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getISPRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesISPRanking();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getWorldNodes(req: Request, res: Response) {
+    try {
+      const worldNodes = await nodesApi.$getWorldNodes();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(worldNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesPerCountry(req: Request, res: Response) {
+    try {
+      const [country]: any[] = await DB.query(
+        `SELECT geo_names.id, geo_names_country.names as country_names
+        FROM geo_names
+        JOIN geo_names geo_names_country on geo_names.id = geo_names_country.id AND geo_names_country.type = 'country'
+        WHERE geo_names.type = 'country_iso_code' AND geo_names.names = ?`,
+        [req.params.country]
+      );
+
+      if (country.length === 0) {
+        res.status(404).send(`This country does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerCountry(country[0].id);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        country: JSON.parse(country[0].country_names),
+        nodes: nodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesPerISP(req: Request, res: Response) {
+    try {
+      const [isp]: any[] = await DB.query(
+        `SELECT geo_names.names as isp_name
+        FROM geo_names
+        WHERE geo_names.type = 'as_organization' AND geo_names.id = ?`,
+        [req.params.isp]
+      );
+
+      if (isp.length === 0) {
+        res.status(404).send(`This ISP does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerISP(req.params.isp);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        isp: JSON.parse(isp[0].isp_name),
+        nodes: nodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesCountries(req: Request, res: Response) {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesCountries();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new NodesRoutes();

backend/src/api/explorer/statistics.api.ts

@@ -0,0 +1,53 @@
+import logger from '../../logger';
+import DB from '../../database';
+import { Common } from '../common';
+
+class StatisticsApi {
+  public async $getStatistics(interval: string | null = null): Promise<any> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
+      tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
+      FROM lightning_stats`;
+
+    if (interval) {
+      query += ` WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` ORDER BY added DESC`;
+
+    try {
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getLatestStatistics(): Promise<any> {
+    try {
+      const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
+      return {
+        latest: rows[0],
+        previous: rows2[0],
+      };
+    } catch (e) {
+      logger.err('$getLatestStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getStatisticsCount(): Promise<number> {
+    try {
+      const [rows]: any = await DB.query(`SELECT count(*) as count FROM lightning_stats`);
+      return rows[0].count;
+    } catch (e) {
+      logger.err('$getLatestStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new StatisticsApi();

backend/src/api/fetch-version.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from "path";
+const { spawnSync } = require('child_process');
+
+function getVersion(): string {
+  const packageJson = fs.readFileSync('package.json').toString();
+  return JSON.parse(packageJson).version;
+}
+
+function getGitCommit(): string {
+  if (process.env.MEMPOOL_COMMIT_HASH) {
+    return process.env.MEMPOOL_COMMIT_HASH;
+  } else {
+    const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+    if (!gitRevParse.error) {
+      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+      if (output) {
+        return output;
+      } else {
+        console.log('Could not fetch git commit: No repo available');
+      }
+    } else if (gitRevParse.error.code === 'ENOENT') {
+      console.log('Could not fetch git commit: Command `git` is unavailable');
+    }
+  }
+  return '?';
+}
+
+const versionInfo = {
+  version: getVersion(),
+  gitCommit: getGitCommit()
+}
+
+fs.writeFileSync(
+  path.join(__dirname, 'version.json'),
+  JSON.stringify(versionInfo, null, 2) + "\n"
+);

backend/src/api/lightning/clightning/clightning-client.ts

@@ -0,0 +1,272 @@
+// Imported from https://github.com/shesek/lightning-client-js
+
+'use strict';
+
+const methods = [
+  'addgossip',
+  'autocleaninvoice',
+  'check',
+  'checkmessage',
+  'close',
+  'connect',
+  'createinvoice',
+  'createinvoicerequest',
+  'createoffer',
+  'createonion',
+  'decode',
+  'decodepay',
+  'delexpiredinvoice',
+  'delinvoice',
+  'delpay',
+  'dev-listaddrs',
+  'dev-rescan-outputs',
+  'disableoffer',
+  'disconnect',
+  'estimatefees',
+  'feerates',
+  'fetchinvoice',
+  'fundchannel',
+  'fundchannel_cancel',
+  'fundchannel_complete',
+  'fundchannel_start',
+  'fundpsbt',
+  'getchaininfo',
+  'getinfo',
+  'getlog',
+  'getrawblockbyheight',
+  'getroute',
+  'getsharedsecret',
+  'getutxout',
+  'help',
+  'invoice',
+  'keysend',
+  'legacypay',
+  'listchannels',
+  'listconfigs',
+  'listforwards',
+  'listfunds',
+  'listinvoices',
+  'listnodes',
+  'listoffers',
+  'listpays',
+  'listpeers',
+  'listsendpays',
+  'listtransactions',
+  'multifundchannel',
+  'multiwithdraw',
+  'newaddr',
+  'notifications',
+  'offer',
+  'offerout',
+  'openchannel_abort',
+  'openchannel_bump',
+  'openchannel_init',
+  'openchannel_signed',
+  'openchannel_update',
+  'pay',
+  'payersign',
+  'paystatus',
+  'ping',
+  'plugin',
+  'reserveinputs',
+  'sendinvoice',
+  'sendonion',
+  'sendonionmessage',
+  'sendpay',
+  'sendpsbt',
+  'sendrawtransaction',
+  'setchannelfee',
+  'signmessage',
+  'signpsbt',
+  'stop',
+  'txdiscard',
+  'txprepare',
+  'txsend',
+  'unreserveinputs',
+  'utxopsbt',
+  'waitanyinvoice',
+  'waitblockheight',
+  'waitinvoice',
+  'waitsendpay',
+  'withdraw'
+];
+
+
+import EventEmitter from 'events';
+import { existsSync, statSync } from 'fs';
+import { createConnection, Socket } from 'net';
+import { homedir } from 'os';
+import path from 'path';
+import { createInterface, Interface } from 'readline';
+import logger from '../../../logger';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
+
+class LightningError extends Error {
+  type: string = 'lightning';
+  message: string = 'lightning-client error';
+
+  constructor(error) {
+    super();
+    this.type = error.type;
+    this.message = error.message;
+  }
+}
+
+const defaultRpcPath = path.join(homedir(), '.lightning')
+  , fStat = (...p) => statSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p))
+
+export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
+  private rpcPath: string;
+  private reconnectWait: number;
+  private reconnectTimeout;
+  private reqcount: number;
+  private client: Socket;
+  private rl: Interface;
+  private clientConnectionPromise: Promise<unknown>;
+
+  constructor(rpcPath = defaultRpcPath) {
+    if (!path.isAbsolute(rpcPath)) {
+      throw new Error('The rpcPath must be an absolute path');
+    }
+
+    if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
+      // network directory provided, use the lightning-rpc within in
+      if (fExists(rpcPath, 'lightning-rpc')) {
+        rpcPath = path.join(rpcPath, 'lightning-rpc');
+      }
+
+      // main data directory provided, default to using the bitcoin mainnet subdirectory
+      // to be removed in v0.2.0
+      else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
+        logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`)
+        logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`)
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+      }
+    }
+
+    logger.debug(`[CLightningClient] Connecting to ${rpcPath}`);
+
+    super();
+    this.rpcPath = rpcPath;
+    this.reconnectWait = 0.5;
+    this.reconnectTimeout = null;
+    this.reqcount = 0;
+
+    const _self = this;
+
+    this.client = createConnection(rpcPath).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+    this.rl = createInterface({ input: this.client }).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+
+    this.clientConnectionPromise = new Promise<void>(resolve => {
+      _self.client.on('connect', () => {
+        logger.info(`[CLightningClient] Lightning client connected`);
+        _self.reconnectWait = 1;
+        resolve();
+      });
+
+      _self.client.on('end', () => {
+        logger.err('[CLightningClient] Lightning client connection closed, reconnecting');
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+
+      _self.client.on('error', error => {
+        logger.err(`[CLightningClient] Lightning client connection error: ${error}`);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+    });
+
+    this.rl.on('line', line => {
+      line = line.trim();
+      if (!line) {
+        return;
+      }
+      const data = JSON.parse(line);
+      // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`);
+      _self.emit('res:' + data.id, data);
+    });
+  }
+
+  increaseWaitTime(): void {
+    if (this.reconnectWait >= 16) {
+      this.reconnectWait = 16;
+    } else {
+      this.reconnectWait *= 2;
+    }
+  }
+
+  reconnect(): void {
+    const _self = this;
+
+    if (this.reconnectTimeout) {
+      return;
+    }
+
+    this.reconnectTimeout = setTimeout(() => {
+      logger.debug('[CLightningClient] Trying to reconnect...');
+
+      _self.client.connect(_self.rpcPath);
+      _self.reconnectTimeout = null;
+    }, this.reconnectWait * 1000);
+  }
+
+  call(method, args = []): Promise<any> {
+    const _self = this;
+
+    const callInt = ++this.reqcount;
+    const sendObj = {
+      jsonrpc: '2.0',
+      method,
+      params: args,
+      id: '' + callInt
+    };
+
+    // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`);
+
+    // Wait for the client to connect
+    return this.clientConnectionPromise
+      .then(() => new Promise((resolve, reject) => {
+        // Wait for a response
+        this.once('res:' + callInt, res => res.error == null
+          ? resolve(res.result)
+          : reject(new LightningError(res.error))
+        );
+
+        // Send the command
+        _self.client.write(JSON.stringify(sendObj));
+      }));
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const listnodes: any[] = await this.call('listnodes');
+    const listchannels: any[] = await this.call('listchannels');
+    const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
+
+    return {
+      nodes: listnodes['nodes'].map(node => convertNode(node)),
+      edges: channelsList,
+    };
+  }
+}
+
+const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
+
+methods.forEach(k => {
+  CLightningClient.prototype[protify(k)] = function (...args: any) {
+    return this.call(k, args);
+  };
+});

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -0,0 +1,147 @@
+import { ILightningApi } from '../lightning-api.interface';
+import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
+import logger from '../../../logger';
+import { Common } from '../../common';
+
+/**
+ * Convert a clightning "listnode" entry to a lnd node entry
+ */
+export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
+  return {
+    alias: clNode.alias ?? '',
+    color: `#${clNode.color ?? ''}`,
+    features: [], // TODO parse and return clNode.feature
+    pub_key: clNode.nodeid,
+    addresses: clNode.addresses?.map((addr) => {
+      let address = addr.address;
+      if (addr.type === 'ipv6') {
+        address = `[${address}]`;
+      }
+      return {
+        network: addr.type,
+        addr: `${address}:${addr.port}`
+      };
+    }) ?? [],
+    last_update: clNode?.last_timestamp ?? 0,
+    custom_records
+  };
+}
+
+/**
+ * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ */
+export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
+  logger.info('Converting clightning nodes and channels to lnd graph format');
+
+  let loggerTimer = new Date().getTime() / 1000;
+  let channelProcessed = 0;
+
+  const consolidatedChannelList: ILightningApi.Channel[] = [];
+  const clChannelsDict = {};
+  const clChannelsDictCount = {};
+
+  for (const clChannel of clChannels) {
+    if (!clChannelsDict[clChannel.short_channel_id]) {
+      clChannelsDict[clChannel.short_channel_id] = clChannel;
+      clChannelsDictCount[clChannel.short_channel_id] = 1;
+    } else {
+      consolidatedChannelList.push(
+        await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id])
+      );
+      delete clChannelsDict[clChannel.short_channel_id];
+      clChannelsDictCount[clChannel.short_channel_id]++;
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    ++channelProcessed;
+  }
+
+  channelProcessed = 0;
+  const keys = Object.keys(clChannelsDict);
+  for (const short_channel_id of keys) {
+    consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    channelProcessed++;
+  }
+
+  return consolidatedChannelList;
+}
+
+/**
+ * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy for both nodes
+ */
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel> {
+  const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
+
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
+  const parts = clChannelA.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
+    capacity: clChannelA.satoshis,
+    last_update: lastUpdate,
+    node1_policy: convertPolicy(clChannelA),
+    node2_policy: convertPolicy(clChannelB),
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannelA.source,
+    node2_pub: clChannelB.source,
+  };
+}
+
+/**
+ * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy of only one node
+ */
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel> {
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
+  const parts = clChannel.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
+    capacity: clChannel.satoshis,
+    last_update: clChannel.last_update ?? 0,
+    node1_policy: convertPolicy(clChannel),
+    node2_policy: null,
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannel.source,
+    node2_pub: clChannel.destination,
+  };
+}
+
+/**
+ * Convert a clightning "listnode" response to a lnd channel policy format
+ */
+function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
+  return {
+    time_lock_delta: clChannel.delay,
+    min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
+    max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
+    fee_base_msat: clChannel.base_fee_millisatoshi,
+    fee_rate_milli_msat: clChannel.fee_per_millionth,
+    disabled: !clChannel.active,
+    last_update: clChannel.last_update ?? 0,
+  };
+}

backend/src/api/lightning/lightning-api-abstract-factory.ts

@@ -0,0 +1,5 @@
+import { ILightningApi } from './lightning-api.interface';
+
+export interface AbstractLightningApi {
+  $getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
+}

backend/src/api/lightning/lightning-api-factory.ts

@@ -0,0 +1,16 @@
+import config from '../../config';
+import CLightningClient from './clightning/clightning-client';
+import { AbstractLightningApi } from './lightning-api-abstract-factory';
+import LndApi from './lnd/lnd-api';
+
+function lightningApiFactory(): AbstractLightningApi {
+  switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
+    case 'cln':
+      return new CLightningClient(config.CLIGHTNING.SOCKET);
+    case 'lnd':
+    default:
+      return new LndApi();
+  }
+}
+
+export default lightningApiFactory();

backend/src/api/lightning/lightning-api.interface.ts

@@ -0,0 +1,92 @@
+export namespace ILightningApi {
+  export interface NetworkInfo {
+    graph_diameter: number;
+    avg_out_degree: number;
+    max_out_degree: number;
+    num_nodes: number;
+    num_channels: number;
+    total_network_capacity: string;
+    avg_channel_size: number;
+    min_channel_size: string;
+    max_channel_size: string;
+    median_channel_size_sat: string;
+    num_zombie_chans: string;
+  }
+
+  export interface NetworkGraph {
+    nodes: Node[];
+    edges: Channel[];
+  }
+
+  export interface Channel {
+    channel_id: string;
+    chan_point: string;
+    last_update: number;
+    node1_pub: string;
+    node2_pub: string;
+    capacity: string;
+    node1_policy: RoutingPolicy | null;
+    node2_policy: RoutingPolicy | null;
+  }
+
+  export interface RoutingPolicy {
+    time_lock_delta: number;
+    min_htlc: string;
+    fee_base_msat: string;
+    fee_rate_milli_msat: string;
+    disabled: boolean;
+    max_htlc_msat: string;
+    last_update: number;
+  }
+
+  export interface Node {
+    last_update: number;
+    pub_key: string;
+    alias: string;
+    addresses: {
+      network: string;
+      addr: string;
+    }[];
+    color: string;
+    features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
+  }
+
+  export interface Info {
+    identity_pubkey: string;
+    alias: string;
+    num_pending_channels: number;
+    num_active_channels: number;
+    num_peers: number;
+    block_height: number;
+    block_hash: string;
+    synced_to_chain: boolean;
+    testnet: boolean;
+    uris: string[];
+    best_header_timestamp: string;
+    version: string;
+    num_inactive_channels: number;
+    chains: {
+      chain: string;
+      network: string;
+    }[];
+    color: string;
+    synced_to_graph: boolean;
+    features: { [key: number]: Feature };
+    commit_hash: string;
+    /** Available on LND since v0.15.0-beta */
+    require_htlc_interceptor?: boolean;
+  }
+
+  export interface Feature {
+    name: string;
+    is_required: boolean;
+    is_known: boolean;
+  }
+
+  export interface ForensicOutput {
+    node?: 1 | 2;
+    type: number;
+    value: number;
+  }
+}

backend/src/api/lightning/lnd/lnd-api.ts

@@ -0,0 +1,41 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { Agent } from 'https';
+import * as fs from 'fs';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import config from '../../../config';
+
+class LndApi implements AbstractLightningApi {
+  axiosConfig: AxiosRequestConfig = {};
+
+  constructor() {
+    if (config.LIGHTNING.ENABLED) {
+      this.axiosConfig = {
+        headers: {
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
+        },
+        httpsAgent: new Agent({
+          ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
+        }),
+        timeout: 10000
+      };
+    }
+  }
+
+  async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
+    return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getInfo(): Promise<ILightningApi.Info> {
+    return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+      .then((response) => response.data);
+  }
+}
+
+export default LndApi;

backend/src/api/liquid/liquid.routes.ts

@@ -0,0 +1,73 @@
+import axios from 'axios';
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import elementsParser from './elements-parser';
+import icons from './icons';
+
+class LiquidRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', this.getAllLiquidIcon)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', this.$getAllFeaturedLiquidAssets)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', this.getLiquidIcon)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', this.$getAssetGroup)
+      ;
+    
+    if (config.DATABASE.ENABLED) {
+      app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', this.$getElementsPegsByMonth)
+        ;
+    }
+  }
+
+
+  private getLiquidIcon(req: Request, res: Response) {
+    const result = icons.getIconByAssetId(req.params.assetId);
+    if (result) {
+      res.setHeader('content-type', 'image/png');
+      res.setHeader('content-length', result.length);
+      res.send(result);
+    } else {
+      res.status(404).send('Asset icon not found');
+    }
+  }
+
+  private getAllLiquidIcon(req: Request, res: Response) {
+    const result = icons.getAllIconIds();
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Asset icons not found');
+    }
+  }
+
+  private async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getAssetGroup(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
+        { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getElementsPegsByMonth(req: Request, res: Response) {
+    try {
+      const pegs = await elementsParser.$getPegDataByMonth();
+      res.json(pegs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new LiquidRoutes();

backend/src/api/mempool-blocks.ts

@@ -2,10 +2,16 @@ import logger from '../logger';
 import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
+import { StaticPool } from 'node-worker-threads-pool';
+import path from 'path';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
   private mempoolBlockDeltas: MempoolBlockDelta[] = [];
+  private makeTemplatesPool = new StaticPool({
+    size: 1,
+    task: path.resolve(__dirname, './tx-selection-worker.js'),
+  });
 
   constructor() {}
 
@@ -71,15 +77,15 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
+
     this.mempoolBlocks = blocks;
     this.mempoolBlockDeltas = deltas;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
-    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     let blockWeight = 0;
     let blockSize = 0;
     let transactions: TransactionExtended[] = [];
@@ -99,7 +105,12 @@ class MempoolBlocks {
     if (transactions.length) {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
-    // Calculate change from previous block states
+
+    return mempoolBlocks;
+  }
+
+  private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
       let removed: string[] = [];
@@ -132,10 +143,26 @@ class MempoolBlocks {
         removed
       });
     }
-    return {
-      blocks: mempoolBlocks,
-      deltas: mempoolBlockDeltas
-    };
+    return mempoolBlockDeltas;
+  }
+
+  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): Promise<void> {
+    const { mempool, blocks } = await this.makeTemplatesPool.exec({ mempool: newMempool, blockLimit, weightLimit, condenseRest });
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
+
+    // copy CPFP info across to main thread's mempool
+    Object.keys(newMempool).forEach((txid) => {
+      if (newMempool[txid] && mempool[txid]) {
+        newMempool[txid].effectiveFeePerVsize = mempool[txid].effectiveFeePerVsize;
+        newMempool[txid].ancestors = mempool[txid].ancestors;
+        newMempool[txid].descendants = mempool[txid].descendants;
+        newMempool[txid].bestDescendant = mempool[txid].bestDescendant;
+        newMempool[txid].cpfpChecked = mempool[txid].cpfpChecked;
+      }
+    });
+
+    this.mempoolBlocks = blocks;
+    this.mempoolBlockDeltas = deltas;
   }
 
   private dataToMempoolBlocks(transactions: TransactionExtended[],

backend/src/api/mempool.ts

@@ -20,6 +20,8 @@ class Mempool {
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
+  private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+    deletedTransactions: TransactionExtended[]) => void) | undefined;
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
@@ -63,6 +65,11 @@ class Mempool {
     this.mempoolChangedCallback = fn;
   }
 
+  public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise<void>) {
+    this.asyncMempoolChangedCallback = fn;
+  }
+
   public getMempool(): { [txid: string]: TransactionExtended } {
     return this.mempoolCache;
   }
@@ -72,6 +79,9 @@ class Mempool {
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
     }
+    if (this.asyncMempoolChangedCallback) {
+      this.asyncMempoolChangedCallback(this.mempoolCache, [], []);
+    }
   }
 
   public async $updateMemPoolInfo() {
@@ -103,12 +113,11 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
-    logger.debug('Updating mempool');
+  public async $updateMempool(): Promise<void> {
+    logger.debug(`Updating mempool...`);
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
     const transactions = await bitcoinApi.$getRawMempool();
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
@@ -124,7 +133,6 @@ class Mempool {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
           this.mempoolCache[txid] = transaction;
-          txCount++;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -133,14 +141,9 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
         } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
 
@@ -194,11 +197,13 @@ class Mempool {
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
+    if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    }
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {

backend/src/api/mining/mining-routes.ts

@@ -0,0 +1,311 @@
+import { Application, Request, Response } from 'express';
+import config from "../../config";
+import logger from '../../logger';
+import audits from '../audit';
+import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
+import BlocksRepository from '../../repositories/BlocksRepository';
+import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
+import HashratesRepository from '../../repositories/HashratesRepository';
+import bitcoinClient from '../bitcoin/bitcoin-client';
+import mining from "./mining";
+
+class MiningRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', this.$getPools)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', this.$getPoolHistoricalHashrate)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', this.$getPoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', this.$getPoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', this.$getPool)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', this.$getPoolsHistoricalHashrate)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', this.$getHistoricalHashrate)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', this.$getDifficultyAdjustments)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', this.$getRewardStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', this.$getHistoricalBlockFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', this.$getHistoricalBlockRewards)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', this.$getHistoricalBlockFeeRates)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
+    ;
+  }
+
+  private async $getPool(req: Request, res: Response): Promise<void> {
+    try {
+      const stats = await mining.$getPoolStat(req.params.slug);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async $getPoolBlocks(req: Request, res: Response) {
+    try {
+      const poolBlocks = await BlocksRepository.$getBlocksByPool(
+        req.params.slug,
+        req.params.height === undefined ? undefined : parseInt(req.params.height, 10),
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(poolBlocks);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async $getPools(req: Request, res: Response) {
+    try {
+      const stats = await mining.$getPoolsStats(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getPoolsHistoricalHashrate(req: Request, res: Response) {
+    try {
+      const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(hashrates);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getPoolHistoricalHashrate(req: Request, res: Response) {
+    try {
+      const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(hashrates);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async $getHistoricalHashrate(req: Request, res: Response) {
+    let currentHashrate = 0, currentDifficulty = 0;
+    try {
+      currentHashrate = await bitcoinClient.getNetworkHashPs();
+      currentDifficulty = await bitcoinClient.getDifficulty();
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
+    }
+
+    try {
+      const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
+      const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json({
+        hashrates: hashrates,
+        difficulty: difficulty,
+        currentHashrate: currentHashrate,
+        currentDifficulty: currentDifficulty,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockFees(req: Request, res: Response) {
+    try {
+      const blockFees = await mining.$getHistoricalBlockFees(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockFees);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockRewards(req: Request, res: Response) {
+    try {
+      const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockRewards);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockFeeRates(req: Request, res: Response) {
+    try {
+      const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockFeeRates);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) {
+    try {
+      const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval);
+      const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        sizes: blockSizes,
+        weights: blockWeights
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getDifficultyAdjustments(req: Request, res: Response) {
+    try {
+      const difficulty = await DifficultyAdjustmentsRepository.$getRawAdjustments(req.params.interval, true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getRewardStats(req: Request, res: Response) {
+    try {
+      const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10));
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(response);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getHistoricalBlockPrediction(req: Request, res: Response) {
+    try {
+      const blockPredictions = await mining.$getBlockPredictionsHistory(req.params.interval);
+      const blockCount = await BlocksAuditsRepository.$getPredictionsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockPredictions.map(prediction => [prediction.time, prediction.height, prediction.match_rate]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAudit(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
+
+      if (!audit) {
+        res.status(404).send(`This block has not been audited.`);
+        return;
+      }
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHeightFromTimestamp(req: Request, res: Response) {
+    try {
+      const timestamp = parseInt(req.params.timestamp, 10);
+      // This will prevent people from entering milliseconds etc.
+      // Block timestamps are allowed to be up to 2 hours off, so 24 hours
+      // will never put the maximum value before the most recent block
+      const nowPlus1day = Math.floor(Date.now() / 1000) + 60 * 60 * 24;
+      // Prevent non-integers that are not seconds
+      if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp > nowPlus1day) {
+        throw new Error(`Invalid timestamp, value must be Unix seconds`);
+      }
+      const result = await BlocksRepository.$getBlockHeightFromTimestamp(
+        timestamp,
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getBlockAuditScores(req: Request, res: Response) {
+    try {
+      let height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+      if (height == null) {
+        height = await BlocksRepository.$mostRecentBlockHeight();
+      }
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await BlocksAuditsRepository.$getBlockAuditScores(height, height - 15));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAuditScore(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash);
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit || 'null');
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new MiningRoutes();

backend/src/api/mining/mining.ts

@@ -1,23 +1,39 @@
-import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
-import BlocksRepository from '../repositories/BlocksRepository';
-import PoolsRepository from '../repositories/PoolsRepository';
-import HashratesRepository from '../repositories/HashratesRepository';
-import bitcoinClient from './bitcoin/bitcoin-client';
-import logger from '../logger';
-import { Common } from './common';
-import loadingIndicators from './loading-indicators';
+import { BlockPrice, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces';
+import BlocksRepository from '../../repositories/BlocksRepository';
+import PoolsRepository from '../../repositories/PoolsRepository';
+import HashratesRepository from '../../repositories/HashratesRepository';
+import bitcoinClient from '../bitcoin/bitcoin-client';
+import logger from '../../logger';
+import { Common } from '../common';
+import loadingIndicators from '../loading-indicators';
 import { escape } from 'mysql2';
+import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
+import config from '../../config';
+import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
+import PricesRepository from '../../repositories/PricesRepository';
 
 class Mining {
+  blocksPriceIndexingRunning = false;
+
   constructor() {
   }
 
+  /**
+   * Get historical block predictions match rate
+   */
+   public async $getBlockPredictionsHistory(interval: string | null = null): Promise<any> {
+    return await BlocksAuditsRepository.$getBlockPredictionsHistory(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
   /**
    * Get historical block total fee
    */
   public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
     return await BlocksRepository.$getHistoricalBlockFees(
-      this.getTimeRange(interval),
+      this.getTimeRange(interval, 5),
       Common.getSqlInterval(interval)
     );
   }
@@ -159,26 +175,25 @@ class Mining {
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
+    const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
 
-    try {
-      const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
-
-      // Run only if:
-      // * lastestRunDate is set to 0 (node backend restart, reorg)
-      // * we started a new week (around Monday midnight)
-      const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
-      if (!runIndexing) {
-        return;
-      }
-    } catch (e) {
-      throw e;
+    // Run only if:
+    // * lastestRunDate is set to 0 (node backend restart, reorg)
+    // * we started a new week (around Monday midnight)
+    const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
+    if (!runIndexing) {
+      return;
     }
 
     try {
+      const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
+
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
       const hashrates: any[] = [];
-      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
-
+ 
       const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
       const lastMondayMidnight = this.getDateMidnight(lastMonday);
       let toTimestamp = lastMondayMidnight.getTime();
@@ -193,7 +208,7 @@ class Mining {
       logger.debug(`Indexing weekly mining pool hashrate`);
       loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
 
-      while (toTimestamp > genesisTimestamp) {
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
         const fromTimestamp = toTimestamp - 604800000;
 
         // Skip already indexed weeks
@@ -203,14 +218,6 @@ class Mining {
           continue;
         }
 
-        // Check if we have blocks for the previous week (which mean that the week
-        // we are currently indexing has complete data)
-        const blockStatsPreviousWeek: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, (fromTimestamp - 604800000) / 1000, (toTimestamp - 604800000) / 1000);
-        if (blockStatsPreviousWeek.blockCount === 0) { // We are done indexing
-          break;
-        }
-
         const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
           null, fromTimestamp / 1000, toTimestamp / 1000);
         const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
@@ -218,34 +225,35 @@ class Mining {
 
         let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
         const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
-        pools = pools.map((pool: any) => {
-          pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
-          pool.share = (pool.blockCount / totalBlocks);
-          return pool;
-        });
-
-        for (const pool of pools) {
-          hashrates.push({
-            hashrateTimestamp: toTimestamp / 1000,
-            avgHashrate: pool['hashrate'],
-            poolId: pool.poolId,
-            share: pool['share'],
-            type: 'weekly',
+        if (totalBlocks > 0) {
+          pools = pools.map((pool: any) => {
+            pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
+            pool.share = (pool.blockCount / totalBlocks);
+            return pool;
           });
-        }
 
-        newlyIndexed += hashrates.length;
-        await HashratesRepository.$saveHashrates(hashrates);
-        hashrates.length = 0;
+          for (const pool of pools) {
+            hashrates.push({
+              hashrateTimestamp: toTimestamp / 1000,
+              avgHashrate: pool['hashrate'] ,
+              poolId: pool.poolId,
+              share: pool['share'],
+              type: 'weekly',
+            });
+          }
+
+          newlyIndexed += hashrates.length;
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
 
         const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
         if (elapsedSeconds > 1) {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
           const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
           const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
-          const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds);
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
           loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
@@ -257,11 +265,14 @@ class Mining {
       }
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
+        logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
+      } else {
+        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
       }
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -270,20 +281,19 @@ class Mining {
    * [INDEXING] Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
-    try {
-      // We only run this once a day around midnight
-      const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
-      const now = new Date().getUTCDate();
-      if (now === latestRunDate) {
-        return;
-      }
-    } catch (e) {
-      throw e;
+    // We only run this once a day around midnight
+    const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
+    const now = new Date().getUTCDate();
+    if (now === latestRunDate) {
+      return;
     }
 
+    const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
     try {
-      const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
-      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
+      const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       const lastMidnight = this.getDateMidnight(new Date());
       let toTimestamp = Math.round(lastMidnight.getTime());
       const hashrates: any[] = [];
@@ -298,27 +308,19 @@ class Mining {
       logger.debug(`Indexing daily network hashrate`);
       loadingIndicators.setProgress('daily-hashrate-indexing', 0);
 
-      while (toTimestamp > genesisTimestamp) {
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
         const fromTimestamp = toTimestamp - 86400000;
 
-        // Skip already indexed weeks
+        // Skip already indexed days
         if (indexedTimestamp.includes(toTimestamp / 1000)) {
           toTimestamp -= 86400000;
           ++totalIndexed;
           continue;
         }
 
-        // Check if we have blocks for the previous day (which mean that the day
-        // we are currently indexing has complete data)
-        const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
-        if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
-          break;
-        }
-
         const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
           null, fromTimestamp / 1000, toTimestamp / 1000);
-        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+        const lastBlockHashrate = blockStats.blockCount === 0 ? 0 : await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
           blockStats.lastBlockHeight);
 
         hashrates.push({
@@ -340,9 +342,8 @@ class Mining {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
           const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
           const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
-          const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds);
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
           loadingIndicators.setProgress('daily-hashrate-indexing', progress);
@@ -354,11 +355,12 @@ class Mining {
       }
 
       // Add genesis block manually
-      if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
+      if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && !indexedTimestamp.includes(genesisTimestamp / 1000)) {
         hashrates.push({
-          hashrateTimestamp: genesisTimestamp,
+          hashrateTimestamp: genesisTimestamp / 1000,
           avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
-          poolId: null,
+          poolId: 0,
+          share: 1,
           type: 'daily',
         });
       }
@@ -368,15 +370,155 @@ class Mining {
 
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
+        logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
+      } else {
+        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
       }
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
 
+  /**
+   * Index difficulty adjustments
+   */
+  public async $indexDifficultyAdjustments(): Promise<void> {
+    const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
+    const indexedHeights = {};
+    for (const height of indexedHeightsArray) {
+      indexedHeights[height] = true;
+    }
+
+    const blocks: any = await BlocksRepository.$getBlocksDifficulty();
+    const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+    let currentDifficulty = genesisBlock.difficulty;
+    let totalIndexed = 0;
+
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
+      await DifficultyAdjustmentsRepository.$saveAdjustments({
+        time: genesisBlock.time,
+        height: 0,
+        difficulty: currentDifficulty,
+        adjustment: 0.0,
+      });
+    }
+
+    const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock();
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) {
+      currentDifficulty = oldestConsecutiveBlock.difficulty;
+    }
+
+    let totalBlockChecked = 0;
+    let timer = new Date().getTime() / 1000;
+
+    for (const block of blocks) {
+      if (block.difficulty !== currentDifficulty) {
+        if (indexedHeights[block.height] === true) { // Already indexed
+          if (block.height >= oldestConsecutiveBlock.height) {
+            currentDifficulty = block.difficulty;
+          }
+          continue;          
+        }
+
+        let adjustment = block.difficulty / currentDifficulty;
+        adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
+
+        await DifficultyAdjustmentsRepository.$saveAdjustments({
+          time: block.time,
+          height: block.height,
+          difficulty: block.difficulty,
+          adjustment: adjustment,
+        });
+
+        totalIndexed++;
+        if (block.height >= oldestConsecutiveBlock.height) {
+          currentDifficulty = block.difficulty;
+        }
+    }
+
+      totalBlockChecked++;
+      const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+      if (elapsedSeconds > 5) {
+        const progress = Math.round(totalBlockChecked / blocks.length * 100);
+        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`);
+        timer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (totalIndexed > 0) {
+      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
+    } else {
+      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`);
+    }
+  }
+
+  /**
+   * Create a link between blocks and the latest price at when they were mined
+   */
+  public async $indexBlockPrices() {
+    if (this.blocksPriceIndexingRunning === true) {
+      return;
+    }
+    this.blocksPriceIndexingRunning = true;
+
+    try {
+      const prices: any[] = await PricesRepository.$getPricesTimesAndId();    
+      const blocksWithoutPrices: any[] = await BlocksRepository.$getBlocksWithoutPrice();
+
+      let totalInserted = 0;
+      const blocksPrices: BlockPrice[] = [];
+
+      for (const block of blocksWithoutPrices) {
+        // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
+        if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
+          blocksPrices.push({
+            height: block.height,
+            priceId: prices[0].id,
+          });
+          continue;
+        }
+        for (const price of prices) {
+          if (block.timestamp < price.time) {
+            blocksPrices.push({
+              height: block.height,
+              priceId: price.id,
+            });
+            break;
+          };
+        }
+
+        if (blocksPrices.length >= 100000) {
+          totalInserted += blocksPrices.length;
+          let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
+          if (blocksWithoutPrices.length > 200000) {
+            logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
+          }
+          logger.debug(logStr);
+          await BlocksRepository.$saveBlockPrices(blocksPrices);
+          blocksPrices.length = 0;
+        }
+      }
+
+      if (blocksPrices.length > 0) {
+        totalInserted += blocksPrices.length;
+        let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
+        if (blocksWithoutPrices.length > 200000) {
+          logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
+        }
+        logger.debug(logStr);
+        await BlocksRepository.$saveBlockPrices(blocksPrices);
+      }
+    } catch (e) {
+      this.blocksPriceIndexingRunning = false;
+      throw e;
+    }
+
+    this.blocksPriceIndexingRunning = false;
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);
@@ -386,18 +528,18 @@ class Mining {
     return date;
   }
 
-  private getTimeRange(interval: string | null): number {
+  private getTimeRange(interval: string | null, scale = 1): number {
     switch (interval) {
-      case '3y': return 43200; // 12h
-      case '2y': return 28800; // 8h
-      case '1y': return 28800; // 8h
-      case '6m': return 10800; // 3h
-      case '3m': return 7200; // 2h
-      case '1m': return 1800; // 30min
-      case '1w': return 300; // 5min
-      case '3d': return 1;
-      case '24h': return 1;
-      default: return 86400; // 24h
+      case '3y': return 43200 * scale; // 12h
+      case '2y': return 28800 * scale; // 8h
+      case '1y': return 28800 * scale; // 8h
+      case '6m': return 10800 * scale; // 3h
+      case '3m': return 7200 * scale; // 2h
+      case '1m': return 1800 * scale; // 30min
+      case '1w': return 300 * scale; // 5min
+      case '3d': return 1 * scale;
+      case '24h': return 1 * scale;
+      default: return 86400 * scale;
     }
   }
 }

backend/src/api/pools-parser.ts

@@ -1,6 +1,7 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
+import BlocksRepository from '../repositories/BlocksRepository';
 
 interface Pool {
   name: string;
@@ -13,10 +14,10 @@ interface Pool {
 class PoolsParser {
   miningPools: any[] = [];
   unknownPool: any = {
-    'name': "Unknown",
-    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
-    'regexes': "[]",
-    'addresses': "[]",
+    'name': 'Unknown',
+    'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
+    'regexes': '[]',
+    'addresses': '[]',
     'slug': 'unknown'
   };
   slugWarnFlag = false;
@@ -24,7 +25,7 @@ class PoolsParser {
   /**
    * Parse the pools.json file, consolidate the data and dump it into the database
    */
-  public async migratePoolsJson(poolsJson: object) {
+  public async migratePoolsJson(poolsJson: object): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -32,7 +33,6 @@ class PoolsParser {
     // First we save every entries without paying attention to pool duplication
     const poolsDuplicated: Pool[] = [];
 
-    logger.debug('Parse coinbase_tags');
     const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
     for (let i = 0; i < coinbaseTags.length; ++i) {
       poolsDuplicated.push({
@@ -43,7 +43,6 @@ class PoolsParser {
         'slug': ''
       });
     }
-    logger.debug('Parse payout_addresses');
     const addressesTags = Object.entries(poolsJson['payout_addresses']);
     for (let i = 0; i < addressesTags.length; ++i) {
       poolsDuplicated.push({
@@ -56,7 +55,6 @@ class PoolsParser {
     }
 
     // Then, we find unique mining pool names
-    logger.debug('Identify unique mining pools');
     const poolNames: string[] = [];
     for (let i = 0; i < poolsDuplicated.length; ++i) {
       if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
@@ -83,6 +81,7 @@ class PoolsParser {
     // Finally, we generate the final consolidated pools data
     const finalPoolDataAdd: Pool[] = [];
     const finalPoolDataUpdate: Pool[] = [];
+    const finalPoolDataRename: Pool[] = [];
     for (let i = 0; i < poolNames.length; ++i) {
       let allAddresses: string[] = [];
       let allRegexes: string[] = [];
@@ -119,11 +118,36 @@ class PoolsParser {
         'slug': slug
       };
 
-      if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
-        finalPoolDataUpdate.push(poolObj);
+      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
+      if (existingPool !== undefined) {
+        // Check if any data was actually updated
+        const equals = (a, b) =>
+          a.length === b.length &&
+          a.every((v, i) => v === b[i]);
+        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
+          finalPoolDataUpdate.push(poolObj);
+        }
       } else {
-        logger.debug(`Add '${finalPoolName}' mining pool`);
-        finalPoolDataAdd.push(poolObj);
+        // Double check that if we're not just renaming a pool (same address same regex)
+        const [poolToRename]: any[] = await DB.query(`
+          SELECT * FROM pools
+          WHERE addresses = ? OR regexes = ?`,
+          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
+        );
+        if (poolToRename && poolToRename.length > 0) {
+          // We're actually renaming an existing pool
+          finalPoolDataRename.push({
+            'name': poolObj.name,
+            'link': poolObj.link,
+            'regexes': allRegexes,
+            'addresses': allAddresses,
+            'slug': slug
+          });
+          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`);
+        } else {
+          logger.debug(`Add '${finalPoolName}' mining pool`);
+          finalPoolDataAdd.push(poolObj);
+        }
       }
 
       this.miningPools.push({
@@ -140,40 +164,70 @@ class PoolsParser {
       return;
     }
 
-    logger.debug(`Update pools table now`);
+    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
+      finalPoolDataRename.length > 0
+    ) {    
+      logger.debug(`Update pools table now`);
 
-    // Add new mining pools into the database
-    let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
-    for (let i = 0; i < finalPoolDataAdd.length; ++i) {
-      queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-      '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-      ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
-    }
-    queryAdd = queryAdd.slice(0, -1) + ';';
+      // Add new mining pools into the database
+      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
+      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
+        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
+        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
+        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
+      }
+      queryAdd = queryAdd.slice(0, -1) + ';';
 
-    // Updated existing mining pools in the database
-    const updateQueries: string[] = [];
-    for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
-      updateQueries.push(`
-        UPDATE pools
-        SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-        regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-        slug='${finalPoolDataUpdate[i].slug}'
-        WHERE name='${finalPoolDataUpdate[i].name}'
-      ;`);
+      // Updated existing mining pools in the database
+      const updateQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
+        updateQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
+          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
+          slug='${finalPoolDataUpdate[i].slug}'
+          WHERE name='${finalPoolDataUpdate[i].name}'
+        ;`);
+      }
+
+      // Rename mining pools
+      const renameQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataRename.length; ++i) {
+        renameQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
+            slug='${finalPoolDataRename[i].slug}'
+          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
+            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
+        ;`);
+      }
+
+      try {
+        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
+          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+        }
+
+        if (finalPoolDataAdd.length > 0) {
+          await DB.query({ sql: queryAdd, timeout: 120000 });
+        }
+        for (const query of updateQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
+        for (const query of renameQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
+        await this.insertUnknownPool();
+        logger.info('Mining pools.json import completed');
+      } catch (e) {
+        logger.err(`Cannot import pools in the database`);
+        throw e;
+      }
     }
 
     try {
-      if (finalPoolDataAdd.length > 0) {
-        await DB.query({ sql: queryAdd, timeout: 120000 });
-      }
-      for (const query of updateQueries) {
-        await DB.query({ sql: query, timeout: 120000 });
-      }
       await this.insertUnknownPool();
-      logger.info('Mining pools.json import completed');
     } catch (e) {
-      logger.err(`Cannot import pools in the database`);
+      logger.err(`Cannot insert unknown pool in the database`);
       throw e;
     }
   }
@@ -201,6 +255,36 @@ class PoolsParser {
       logger.err('Unable to insert "Unknown" mining pool');
     }
   }
+
+  /**
+   * Delete blocks which needs to be reindexed
+   */
+   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+    if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
+      return;
+    }
+
+    const blockCount = await BlocksRepository.$blockCount(null, null);
+    if (blockCount === 0) {
+      return;
+    }
+
+    for (const updatedPool of finalPoolDataUpdate) {
+      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
+      if (pool.length > 0) {
+        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`);
+        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
+      }
+    }
+
+    // Ignore early days of Bitcoin as there were not mining pool yet
+    logger.notice('Deleting blocks with unknown mining pool from height 130635 for future re-indexing');
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+
+    logger.notice('Truncating hashrates for future re-indexing');
+    await DB.query(`DELETE FROM hashrates`);
+  }
 }
 
 export default new PoolsParser();

backend/src/api/statistics/statistics-api.ts

@@ -1,160 +1,11 @@
-import memPool from './mempool';
-import DB from '../database';
-import logger from '../logger';
+import DB from '../../database';
+import logger from '../../logger';
+import { Statistic, OptimizedStatistic } from '../../mempool.interfaces';
 
-import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces';
-import config from '../config';
-import { Common } from './common';
-
-class Statistics {
-  protected intervalTimer: NodeJS.Timer | undefined;
-  protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined;
+class StatisticsApi {
   protected queryTimeout = 120000;
 
-  public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) {
-    this.newStatisticsEntryCallback = fn;
-  }
-
-  constructor() { }
-
-  public startStatistics(): void {
-    logger.info('Starting statistics service');
-
-    const now = new Date();
-    const nextInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(),
-      Math.floor(now.getMinutes() / 1) * 1 + 1, 0, 0);
-    const difference = nextInterval.getTime() - now.getTime();
-
-    setTimeout(() => {
-      this.runStatistics();
-      this.intervalTimer = setInterval(() => {
-        this.runStatistics();
-      }, 1 * 60 * 1000);
-    }, difference);
-  }
-
-  private async runStatistics(): Promise<void> {
-    if (!memPool.isInSync()) {
-      return;
-    }
-    const currentMempool = memPool.getMempool();
-    const txPerSecond = memPool.getTxPerSecond();
-    const vBytesPerSecond = memPool.getVBytesPerSecond();
-
-    logger.debug('Running statistics');
-
-    let memPoolArray: TransactionExtended[] = [];
-    for (const i in currentMempool) {
-      if (currentMempool.hasOwnProperty(i)) {
-        memPoolArray.push(currentMempool[i]);
-      }
-    }
-    // Remove 0 and undefined
-    memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
-
-    if (!memPoolArray.length) {
-      try {
-        const insertIdZeroed = await this.$createZeroedStatistic();
-        if (this.newStatisticsEntryCallback && insertIdZeroed) {
-          const newStats = await this.$get(insertIdZeroed);
-          if (newStats) {
-            this.newStatisticsEntryCallback(newStats);
-          }
-        }
-      } catch (e) {
-        logger.err('Unable to insert zeroed statistics. ' + e);
-      }
-      return;
-    }
-
-    memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize);
-    const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
-    const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
-
-    const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
-      250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
-
-    const weightVsizeFees: { [feePerWU: number]: number } = {};
-    const lastItem = logFees.length - 1;
-
-    memPoolArray.forEach((transaction) => {
-      for (let i = 0; i < logFees.length; i++) {
-        if (
-          (Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1]))
-          ||
-          (!Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1]))
-        ) {
-          if (weightVsizeFees[logFees[i]]) {
-            weightVsizeFees[logFees[i]] += transaction.vsize;
-          } else {
-            weightVsizeFees[logFees[i]] = transaction.vsize;
-          }
-          break;
-        }
-      }
-    });
-
-    try {
-      const insertId = await this.$create({
-        added: 'NOW()',
-        unconfirmed_transactions: memPoolArray.length,
-        tx_per_second: txPerSecond,
-        vbytes_per_second: Math.round(vBytesPerSecond),
-        mempool_byte_weight: totalWeight,
-        total_fee: totalFee,
-        fee_data: '',
-        vsize_1: weightVsizeFees['1'] || 0,
-        vsize_2: weightVsizeFees['2'] || 0,
-        vsize_3: weightVsizeFees['3'] || 0,
-        vsize_4: weightVsizeFees['4'] || 0,
-        vsize_5: weightVsizeFees['5'] || 0,
-        vsize_6: weightVsizeFees['6'] || 0,
-        vsize_8: weightVsizeFees['8'] || 0,
-        vsize_10: weightVsizeFees['10'] || 0,
-        vsize_12: weightVsizeFees['12'] || 0,
-        vsize_15: weightVsizeFees['15'] || 0,
-        vsize_20: weightVsizeFees['20'] || 0,
-        vsize_30: weightVsizeFees['30'] || 0,
-        vsize_40: weightVsizeFees['40'] || 0,
-        vsize_50: weightVsizeFees['50'] || 0,
-        vsize_60: weightVsizeFees['60'] || 0,
-        vsize_70: weightVsizeFees['70'] || 0,
-        vsize_80: weightVsizeFees['80'] || 0,
-        vsize_90: weightVsizeFees['90'] || 0,
-        vsize_100: weightVsizeFees['100'] || 0,
-        vsize_125: weightVsizeFees['125'] || 0,
-        vsize_150: weightVsizeFees['150'] || 0,
-        vsize_175: weightVsizeFees['175'] || 0,
-        vsize_200: weightVsizeFees['200'] || 0,
-        vsize_250: weightVsizeFees['250'] || 0,
-        vsize_300: weightVsizeFees['300'] || 0,
-        vsize_350: weightVsizeFees['350'] || 0,
-        vsize_400: weightVsizeFees['400'] || 0,
-        vsize_500: weightVsizeFees['500'] || 0,
-        vsize_600: weightVsizeFees['600'] || 0,
-        vsize_700: weightVsizeFees['700'] || 0,
-        vsize_800: weightVsizeFees['800'] || 0,
-        vsize_900: weightVsizeFees['900'] || 0,
-        vsize_1000: weightVsizeFees['1000'] || 0,
-        vsize_1200: weightVsizeFees['1200'] || 0,
-        vsize_1400: weightVsizeFees['1400'] || 0,
-        vsize_1600: weightVsizeFees['1600'] || 0,
-        vsize_1800: weightVsizeFees['1800'] || 0,
-        vsize_2000: weightVsizeFees['2000'] || 0,
-      });
-
-      if (this.newStatisticsEntryCallback && insertId) {
-        const newStats = await this.$get(insertId);
-        if (newStats) {
-          this.newStatisticsEntryCallback(newStats);
-        }
-      }
-    } catch (e) {
-      logger.err('Unable to insert statistics. ' + e);
-    }
-  }
-
-  private async $createZeroedStatistic(): Promise<number | undefined> {
+  public async $createZeroedStatistic(): Promise<number | undefined> {
     try {
       const query = `INSERT INTO statistics(
               added,
@@ -212,7 +63,7 @@ class Statistics {
     }
   }
 
-  private async $create(statistics: Statistic): Promise<number | undefined> {
+  public async $create(statistics: Statistic): Promise<number | undefined> {
     try {
       const query = `INSERT INTO statistics(
               added,
@@ -413,7 +264,7 @@ class Statistics {
       ORDER BY statistics.added DESC;`;
   }
 
-  private async $get(id: number): Promise<OptimizedStatistic | undefined> {
+  public async $get(id: number): Promise<OptimizedStatistic | undefined> {
     try {
       const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
       const [rows] = await DB.query(query, [id]);
@@ -574,7 +425,6 @@ class Statistics {
       };
     });
   }
-
 }
 
-export default new Statistics();
+export default new StatisticsApi();

backend/src/api/statistics/statistics.routes.ts

@@ -0,0 +1,67 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import statisticsApi from './statistics-api';
+
+class StatisticsRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', this.$getStatisticsByTime.bind(this, '2h'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', this.$getStatisticsByTime.bind(this, '24h'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', this.$getStatisticsByTime.bind(this, '1w'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', this.$getStatisticsByTime.bind(this, '1m'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', this.$getStatisticsByTime.bind(this, '3m'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', this.$getStatisticsByTime.bind(this, '6m'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', this.$getStatisticsByTime.bind(this, '1y'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y'))
+    ;
+  }
+
+  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y', req: Request, res: Response) {
+    res.header('Pragma', 'public');
+    res.header('Cache-control', 'public');
+    res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+
+    try {
+      let result;
+      switch (time as string) {
+        case '2h':
+          result = await statisticsApi.$list2H();
+          res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+          break;
+        case '24h':
+          result = await statisticsApi.$list24H();
+          res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+          break;
+        case '1w':
+          result = await statisticsApi.$list1W();
+          break;
+        case '1m':
+          result = await statisticsApi.$list1M();
+          break;
+        case '3m':
+          result = await statisticsApi.$list3M();
+          break;
+        case '6m':
+          result = await statisticsApi.$list6M();
+          break;
+        case '1y':
+          result = await statisticsApi.$list1Y();
+          break;
+        case '2y':
+          result = await statisticsApi.$list2Y();
+          break;
+        case '3y':
+          result = await statisticsApi.$list3Y();
+          break;
+        default:
+          result = await statisticsApi.$list2H();
+      }
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new StatisticsRoutes();

backend/src/api/statistics/statistics.ts

@@ -0,0 +1,153 @@
+import memPool from '../mempool';
+import logger from '../../logger';
+import { TransactionExtended, OptimizedStatistic } from '../../mempool.interfaces';
+import { Common } from '../common';
+import statisticsApi from './statistics-api';
+
+class Statistics {
+  protected intervalTimer: NodeJS.Timer | undefined;
+  protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined;
+
+  public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) {
+    this.newStatisticsEntryCallback = fn;
+  }
+
+  public startStatistics(): void {
+    logger.info('Starting statistics service');
+
+    const now = new Date();
+    const nextInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(),
+      Math.floor(now.getMinutes() / 1) * 1 + 1, 0, 0);
+    const difference = nextInterval.getTime() - now.getTime();
+
+    setTimeout(() => {
+      this.runStatistics();
+      this.intervalTimer = setInterval(() => {
+        this.runStatistics();
+      }, 1 * 60 * 1000);
+    }, difference);
+  }
+
+  private async runStatistics(): Promise<void> {
+    if (!memPool.isInSync()) {
+      return;
+    }
+    const currentMempool = memPool.getMempool();
+    const txPerSecond = memPool.getTxPerSecond();
+    const vBytesPerSecond = memPool.getVBytesPerSecond();
+
+    logger.debug('Running statistics');
+
+    let memPoolArray: TransactionExtended[] = [];
+    for (const i in currentMempool) {
+      if (currentMempool.hasOwnProperty(i)) {
+        memPoolArray.push(currentMempool[i]);
+      }
+    }
+    // Remove 0 and undefined
+    memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
+
+    if (!memPoolArray.length) {
+      try {
+        const insertIdZeroed = await statisticsApi.$createZeroedStatistic();
+        if (this.newStatisticsEntryCallback && insertIdZeroed) {
+          const newStats = await statisticsApi.$get(insertIdZeroed);
+          if (newStats) {
+            this.newStatisticsEntryCallback(newStats);
+          }
+        }
+      } catch (e) {
+        logger.err('Unable to insert zeroed statistics. ' + e);
+      }
+      return;
+    }
+
+    memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize);
+    const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
+    const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
+
+    const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
+      250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
+
+    const weightVsizeFees: { [feePerWU: number]: number } = {};
+    const lastItem = logFees.length - 1;
+
+    memPoolArray.forEach((transaction) => {
+      for (let i = 0; i < logFees.length; i++) {
+        if (
+          (Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1]))
+          ||
+          (!Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1]))
+        ) {
+          if (weightVsizeFees[logFees[i]]) {
+            weightVsizeFees[logFees[i]] += transaction.vsize;
+          } else {
+            weightVsizeFees[logFees[i]] = transaction.vsize;
+          }
+          break;
+        }
+      }
+    });
+
+    try {
+      const insertId = await statisticsApi.$create({
+        added: 'NOW()',
+        unconfirmed_transactions: memPoolArray.length,
+        tx_per_second: txPerSecond,
+        vbytes_per_second: Math.round(vBytesPerSecond),
+        mempool_byte_weight: totalWeight,
+        total_fee: totalFee,
+        fee_data: '',
+        vsize_1: weightVsizeFees['1'] || 0,
+        vsize_2: weightVsizeFees['2'] || 0,
+        vsize_3: weightVsizeFees['3'] || 0,
+        vsize_4: weightVsizeFees['4'] || 0,
+        vsize_5: weightVsizeFees['5'] || 0,
+        vsize_6: weightVsizeFees['6'] || 0,
+        vsize_8: weightVsizeFees['8'] || 0,
+        vsize_10: weightVsizeFees['10'] || 0,
+        vsize_12: weightVsizeFees['12'] || 0,
+        vsize_15: weightVsizeFees['15'] || 0,
+        vsize_20: weightVsizeFees['20'] || 0,
+        vsize_30: weightVsizeFees['30'] || 0,
+        vsize_40: weightVsizeFees['40'] || 0,
+        vsize_50: weightVsizeFees['50'] || 0,
+        vsize_60: weightVsizeFees['60'] || 0,
+        vsize_70: weightVsizeFees['70'] || 0,
+        vsize_80: weightVsizeFees['80'] || 0,
+        vsize_90: weightVsizeFees['90'] || 0,
+        vsize_100: weightVsizeFees['100'] || 0,
+        vsize_125: weightVsizeFees['125'] || 0,
+        vsize_150: weightVsizeFees['150'] || 0,
+        vsize_175: weightVsizeFees['175'] || 0,
+        vsize_200: weightVsizeFees['200'] || 0,
+        vsize_250: weightVsizeFees['250'] || 0,
+        vsize_300: weightVsizeFees['300'] || 0,
+        vsize_350: weightVsizeFees['350'] || 0,
+        vsize_400: weightVsizeFees['400'] || 0,
+        vsize_500: weightVsizeFees['500'] || 0,
+        vsize_600: weightVsizeFees['600'] || 0,
+        vsize_700: weightVsizeFees['700'] || 0,
+        vsize_800: weightVsizeFees['800'] || 0,
+        vsize_900: weightVsizeFees['900'] || 0,
+        vsize_1000: weightVsizeFees['1000'] || 0,
+        vsize_1200: weightVsizeFees['1200'] || 0,
+        vsize_1400: weightVsizeFees['1400'] || 0,
+        vsize_1600: weightVsizeFees['1600'] || 0,
+        vsize_1800: weightVsizeFees['1800'] || 0,
+        vsize_2000: weightVsizeFees['2000'] || 0,
+      });
+
+      if (this.newStatisticsEntryCallback && insertId) {
+        const newStats = await statisticsApi.$get(insertId);
+        if (newStats) {
+          this.newStatisticsEntryCallback(newStats);
+        }
+      }
+    } catch (e) {
+      logger.err('Unable to insert statistics. ' + e);
+    }
+  }
+}
+
+export default new Statistics();

backend/src/api/tx-selection-worker.ts

@@ -0,0 +1,338 @@
+import config from '../config';
+import logger from '../logger';
+import { TransactionExtended, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces';
+import { PairingHeap } from '../utils/pairing-heap';
+import { Common } from './common';
+import { parentPort } from 'worker_threads';
+
+if (parentPort) {
+  parentPort.on('message', (params: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null, condenseRest: boolean}) => {
+    const { mempool, blocks } = makeBlockTemplates(params);
+
+    // return the result to main thread.
+    if (parentPort) {
+     parentPort.postMessage({ mempool, blocks });
+    }
+  });
+}
+
+/*
+* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+*
+* blockLimit: number of blocks to build in total.
+* weightLimit: maximum weight of transactions to consider using the selection algorithm.
+*              if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate 
+* condenseRest: whether to ignore excess transactions or append them to the final block.
+*/
+function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit?: number | null, condenseRest?: boolean | null })
+  : { mempool: { [txid: string]: TransactionExtended }, blocks: MempoolBlockWithTransactions[] } {
+  const start = Date.now();
+  const auditPool: { [txid: string]: AuditTransaction } = {};
+  const mempoolArray: AuditTransaction[] = [];
+  const restOfArray: TransactionExtended[] = [];
+  
+  let weight = 0;
+  const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity;
+  // grab the top feerate txs up to maxWeight
+  Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+    weight += tx.weight;
+    if (weight >= maxWeight) {
+      restOfArray.push(tx);
+      return;
+    }
+    // initializing everything up front helps V8 optimize property access later
+    auditPool[tx.txid] = {
+      txid: tx.txid,
+      fee: tx.fee,
+      size: tx.size,
+      weight: tx.weight,
+      feePerVsize: tx.feePerVsize,
+      vin: tx.vin,
+      relativesSet: false,
+      ancestorMap: new Map<string, AuditTransaction>(),
+      children: new Set<AuditTransaction>(),
+      ancestorFee: 0,
+      ancestorWeight: 0,
+      score: 0,
+      used: false,
+      modified: false,
+      modifiedNode: null,
+    };
+    mempoolArray.push(auditPool[tx.txid]);
+  });
+
+  // Build relatives graph & calculate ancestor scores
+  for (const tx of mempoolArray) {
+    if (!tx.relativesSet) {
+      setRelatives(tx, auditPool);
+    }
+  }
+
+  // Sort by descending ancestor score
+  mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+
+  // Build blocks by greedily choosing the highest feerate package
+  // (i.e. the package rooted in the transaction with the best ancestor score)
+  const blocks: MempoolBlockWithTransactions[] = [];
+  let blockWeight = 4000;
+  let blockSize = 0;
+  let transactions: AuditTransaction[] = [];
+  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+  let overflow: AuditTransaction[] = [];
+  let failures = 0;
+  let top = 0;
+  while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) {
+    // skip invalid transactions
+    while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
+      top++;
+    }
+
+    // Select best next package
+    let nextTx;
+    const nextPoolTx = mempoolArray[top];
+    const nextModifiedTx = modified.peek();
+    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+      nextTx = nextPoolTx;
+      top++;
+    } else {
+      modified.pop();
+      if (nextModifiedTx) {
+        nextTx = nextModifiedTx;
+        nextTx.modifiedNode = undefined;
+      }
+    }
+
+    if (nextTx && !nextTx?.used) {
+      // Check if the package fits into this block
+      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+        const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
+        const descendants: AuditTransaction[] = [];
+        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+        const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
+        const used: AuditTransaction[] = [];
+        while (sortedTxSet.length) {
+          const ancestor = sortedTxSet.pop();
+          const mempoolTx = mempool[ancestor.txid];
+          ancestor.used = true;
+          ancestor.usedBy = nextTx.txid;
+          // update original copy of this tx with effective fee rate & relatives data
+          mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+          mempoolTx.ancestors = sortedTxSet.map((a) => {
+            return {
+              txid: a.txid,
+              fee: a.fee,
+              weight: a.weight,
+            };
+          }).reverse();
+          mempoolTx.descendants = descendants.map((a) => {
+            return {
+              txid: a.txid,
+              fee: a.fee,
+              weight: a.weight,
+            };
+          });
+          descendants.push(ancestor);
+          mempoolTx.cpfpChecked = true;
+          transactions.push(ancestor);
+          blockSize += ancestor.size;
+          blockWeight += ancestor.weight;
+          used.push(ancestor);
+        }
+
+        // remove these as valid package ancestors for any descendants remaining in the mempool
+        if (used.length) {
+          used.forEach(tx => {
+            updateDescendants(tx, auditPool, modified);
+          });
+        }
+
+        failures = 0;
+      } else {
+        // hold this package in an overflow list while we check for smaller options
+        overflow.push(nextTx);
+        failures++;
+      }
+    }
+
+    // this block is full
+    const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+    const queueEmpty = top >= mempoolArray.length && modified.isEmpty();
+    if ((exceededPackageTries || queueEmpty) && (!condenseRest || blocks.length < blockLimit - 1)) {
+      // construct this block
+      if (transactions.length) {
+        blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+      }
+      // reset for the next block
+      transactions = [];
+      blockSize = 0;
+      blockWeight = 4000;
+
+      // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+      for (const overflowTx of overflow.reverse()) {
+        if (overflowTx.modified) {
+          overflowTx.modifiedNode = modified.add(overflowTx);
+        } else {
+          top--;
+          mempoolArray[top] = overflowTx;
+        }
+      }
+      overflow = [];
+    }
+  }
+  if (condenseRest) {
+    // pack any leftover transactions into the last block
+    for (const tx of overflow) {
+      if (!tx || tx?.used) {
+        continue;
+      }
+      blockWeight += tx.weight;
+      blockSize += tx.size;
+      const mempoolTx = mempool[tx.txid];
+      // update original copy of this tx with effective fee rate & relatives data
+      mempoolTx.effectiveFeePerVsize = tx.score;
+      mempoolTx.ancestors = (Array.from(tx.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
+        return {
+          txid: a.txid,
+          fee: a.fee,
+          weight: a.weight,
+        };
+      });
+      mempoolTx.bestDescendant = null;
+      mempoolTx.cpfpChecked = true;
+      transactions.push(tx);
+      tx.used = true;
+    }
+    const blockTransactions = transactions.map(t => mempool[t.txid]);
+    restOfArray.forEach(tx => {
+      blockWeight += tx.weight;
+      blockSize += tx.size;
+      tx.effectiveFeePerVsize = tx.feePerVsize;
+      tx.cpfpChecked = false;
+      tx.ancestors = [];
+      tx.bestDescendant = null;
+      blockTransactions.push(tx);
+    });
+    if (blockTransactions.length) {
+      blocks.push(dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length));
+    }
+    transactions = [];
+  } else if (transactions.length) {
+    blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+  }
+
+  const end = Date.now();
+  const time = end - start;
+  logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+  return {
+    mempool,
+    blocks
+  };
+}
+
+// traverse in-mempool ancestors
+// recursion unavoidable, but should be limited to depth < 25 by mempool policy
+function setRelatives(
+  tx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+): void {
+  for (const parent of tx.vin) {
+    const parentTx = mempool[parent.txid];
+    if (parentTx && !tx.ancestorMap?.has(parent.txid)) {
+      tx.ancestorMap.set(parent.txid, parentTx);
+      parentTx.children.add(tx);
+      // visit each node only once
+      if (!parentTx.relativesSet) {
+        setRelatives(parentTx, mempool);
+      }
+      parentTx.ancestorMap.forEach((ancestor) => {
+        tx.ancestorMap.set(ancestor.txid, ancestor);
+      });
+    }
+  };
+  tx.ancestorFee = tx.fee || 0;
+  tx.ancestorWeight = tx.weight || 0;
+  tx.ancestorMap.forEach((ancestor) => {
+    tx.ancestorFee += ancestor.fee;
+    tx.ancestorWeight += ancestor.weight;
+  });
+  tx.score = tx.ancestorFee / ((tx.ancestorWeight / 4) || 1);
+  tx.relativesSet = true;
+}
+
+// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+// avoids recursion to limit call stack depth
+function updateDescendants(
+  rootTx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+  modified: PairingHeap<AuditTransaction>,
+): void {
+  const descendantSet: Set<AuditTransaction> = new Set();
+  // stack of nodes left to visit
+  const descendants: AuditTransaction[] = [];
+  let descendantTx;
+  let tmpScore;
+  rootTx.children.forEach(childTx => {
+    if (!descendantSet.has(childTx)) {
+      descendants.push(childTx);
+      descendantSet.add(childTx);
+    }
+  });
+  while (descendants.length) {
+    descendantTx = descendants.pop();
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+      // remove tx as ancestor
+      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorFee -= rootTx.fee;
+      descendantTx.ancestorWeight -= rootTx.weight;
+      tmpScore = descendantTx.score;
+      descendantTx.score = descendantTx.ancestorFee / (descendantTx.ancestorWeight / 4);
+
+      if (!descendantTx.modifiedNode) {
+        descendantTx.modified = true;
+        descendantTx.modifiedNode = modified.add(descendantTx);
+      } else {
+        // rebalance modified heap if score has changed
+        if (descendantTx.score < tmpScore) {
+          modified.decreasePriority(descendantTx.modifiedNode);
+        } else if (descendantTx.score > tmpScore) {
+          modified.increasePriority(descendantTx.modifiedNode);
+        }
+      }
+
+      // add this node's children to the stack
+      descendantTx.children.forEach(childTx => {
+        // visit each node only once
+        if (!descendantSet.has(childTx)) {
+          descendants.push(childTx);
+          descendantSet.add(childTx);
+        }
+      });
+    }
+  }
+}
+
+function dataToMempoolBlocks(transactions: TransactionExtended[],
+  blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
+  let rangeLength = 4;
+  if (blocksIndex === 0) {
+    rangeLength = 8;
+  }
+  if (transactions.length > 4000) {
+    rangeLength = 6;
+  } else if (transactions.length > 10000) {
+    rangeLength = 8;
+  }
+  return {
+    blockSize: blockSize,
+    blockVSize: blockWeight / 4,
+    nTx: transactions.length,
+    totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
+    medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+    feeRange: Common.getFeesInRange(transactions, rangeLength),
+    transactionIds: transactions.map((tx) => tx.txid),
+    transactions: transactions.map((tx) => Common.stripTransaction(tx)),
+  };
+}

backend/src/api/websocket-handler.ts

@@ -1,7 +1,9 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
-import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
-  OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
+import {
+  BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+  OptimizedStatistic, ILoadingIndicators, IConversionRates
+} from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
@@ -14,6 +16,9 @@ import transactionUtils from './transaction-utils';
 import rbfCache from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import Audit from './audit';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -164,7 +169,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -179,7 +184,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -192,7 +197,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -224,7 +229,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -239,13 +244,18 @@ class WebsocketHandler {
     });
   }
 
-  handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
-    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) {
+  async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    mempoolBlocks.updateMempoolBlocks(newMempool);
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(newMempool);
+    }
+
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     const mempoolInfo = memPool.getMempoolInfo();
@@ -255,7 +265,7 @@ class WebsocketHandler {
     memPool.handleRbfTransactions(rbfTransactions);
     const recommendedFees = feeApi.getRecommendedFee();
 
-    this.wss.clients.forEach(async (client: WebSocket) => {
+    this.wss.clients.forEach(async (client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -400,37 +410,71 @@ class WebsocketHandler {
       }
     });
   }
-
-  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
+ 
+  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    let mBlocks: undefined | MempoolBlock[];
-    let mBlockDeltas: undefined | MempoolBlockDelta[];
-    let matchRate = 0;
     const _memPool = memPool.getMempool();
-    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
 
-    if (_mempoolBlocks[0]) {
-      const matches: string[] = [];
-      for (const txId of txIds) {
-        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
-          matches.push(txId);
-        }
-        delete _memPool[txId];
-      }
-
-      matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
+    if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+      await mempoolBlocks.makeBlockTemplates(_memPool, 2);
+    } else {
       mempoolBlocks.updateMempoolBlocks(_memPool);
-      mBlocks = mempoolBlocks.getMempoolBlocks();
-      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     }
 
-    if (block.extras) {
-      block.extras.matchRate = matchRate;
+    if (Common.indexingEnabled() && memPool.isInSync()) {
+      const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+
+      const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
+      const matchRate = Math.round(score * 100 * 100) / 100;
+
+      const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
+        return {
+          txid: tx.txid,
+          vsize: tx.vsize,
+          fee: tx.fee ? Math.round(tx.fee) : 0,
+          value: tx.value,
+        };
+      }) : [];
+
+      BlocksSummariesRepository.$saveTemplate({
+        height: block.height,
+        template: {
+          id: block.id,
+          transactions: stripped
+        }
+      });
+
+      BlocksAuditsRepository.$saveAudit({
+        time: block.timestamp,
+        height: block.height,
+        hash: block.id,
+        addedTxs: added,
+        missingTxs: censored,
+        freshTxs: fresh,
+        matchRate: matchRate,
+      });
+
+      if (block.extras) {
+        block.extras.matchRate = matchRate;
+      }
     }
 
+    // Update mempool to remove transactions included in the new block
+    for (const txId of txIds) {
+      delete _memPool[txId];
+    }
+
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.makeBlockTemplates(_memPool, 8, null, true);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(_memPool);
+    }
+    const mBlocks = mempoolBlocks.getMempoolBlocks();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
     const da = difficultyAdjustment.getDifficultyAdjustment();
     const fees = feeApi.getRecommendedFee();
 

backend/src/config.ts

@@ -1,7 +1,10 @@
-const configFile = require('../mempool-config.json');
+const configFromFile = require(
+    process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
+);
 
 interface IConfig {
   MEMPOOL: {
+    ENABLED: boolean;
     NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
@@ -15,6 +18,7 @@ interface IConfig {
     INITIAL_BLOCKS_AMOUNT: number;
     MEMPOOL_BLOCKS_AMOUNT: number;
     INDEXING_BLOCKS_AMOUNT: number;
+    BLOCKS_SUMMARIES_INDEXING: boolean;
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
@@ -22,10 +26,34 @@ interface IConfig {
     EXTERNAL_RETRY_INTERVAL: number;
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+    AUTOMATIC_BLOCK_REINDEXING: boolean;
+    POOLS_JSON_URL: string,
+    POOLS_JSON_TREE_URL: string,
+    ADVANCED_GBT_AUDIT: boolean;
+    ADVANCED_GBT_MEMPOOL: boolean;
+    TRANSACTION_INDEXING: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
   };
+  LIGHTNING: {
+    ENABLED: boolean;
+    BACKEND: 'lnd' | 'cln' | 'ldk';
+    TOPOLOGY_FOLDER: string;
+    STATS_REFRESH_INTERVAL: number;
+    GRAPH_REFRESH_INTERVAL: number;
+    LOGGER_UPDATE_INTERVAL: number;
+    FORENSICS_INTERVAL: number;
+    FORENSICS_RATE_LIMIT: number;
+  };
+  LND: {
+    TLS_CERT_PATH: string;
+    MACAROON_PATH: string;
+    REST_API_URL: string;
+  };
+  CLIGHTNING: {
+    SOCKET: string;
+  };
   ELECTRUM: {
     HOST: string;
     PORT: number;
@@ -87,10 +115,17 @@ interface IConfig {
     BISQ_URL: string;
     BISQ_ONION: string;
   };
+  MAXMIND: {
+    ENABLED: boolean;
+    GEOLITE2_CITY: string;
+    GEOLITE2_ASN: string;
+    GEOIP2_ISP: string;
+  },
 }
 
 const defaults: IConfig = {
   'MEMPOOL': {
+    'ENABLED': true,
     'NETWORK': 'mainnet',
     'BACKEND': 'none',
     'HTTP_PORT': 8999,
@@ -104,6 +139,7 @@ const defaults: IConfig = {
     'INITIAL_BLOCKS_AMOUNT': 8,
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
+    'BLOCKS_SUMMARIES_INDEXING': false,
     'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [],
@@ -111,6 +147,12 @@ const defaults: IConfig = {
     'EXTERNAL_RETRY_INTERVAL': 0,
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
+    'AUTOMATIC_BLOCK_REINDEXING': false,
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'ADVANCED_GBT_AUDIT': false,
+    'ADVANCED_GBT_MEMPOOL': false,
+    'TRANSACTION_INDEXING': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -156,6 +198,24 @@ const defaults: IConfig = {
     'ENABLED': false,
     'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
   },
+  'LIGHTNING': {
+    'ENABLED': false,
+    'BACKEND': 'lnd',
+    'TOPOLOGY_FOLDER': '',
+    'STATS_REFRESH_INTERVAL': 600,
+    'GRAPH_REFRESH_INTERVAL': 600,
+    'LOGGER_UPDATE_INTERVAL': 30,
+    'FORENSICS_INTERVAL': 43200,
+    'FORENSICS_RATE_LIMIT': 20,
+  },
+  'LND': {
+    'TLS_CERT_PATH': '',
+    'MACAROON_PATH': '',
+    'REST_API_URL': 'https://localhost:8080',
+  },
+  'CLIGHTNING': {
+    'SOCKET': '',
+  },
   'SOCKS5PROXY': {
     'ENABLED': false,
     'USE_ONION': true,
@@ -164,18 +224,24 @@ const defaults: IConfig = {
     'USERNAME': '',
     'PASSWORD': ''
   },
-  "PRICE_DATA_SERVER": {
+  'PRICE_DATA_SERVER': {
     'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
     'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
   },
-  "EXTERNAL_DATA_SERVER": {
+  'EXTERNAL_DATA_SERVER': {
     'MEMPOOL_API': 'https://mempool.space/api/v1',
     'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
     'LIQUID_API': 'https://liquid.network/api/v1',
     'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
     'BISQ_URL': 'https://bisq.markets/api',
     'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
-  }
+  },
+  'MAXMIND': {
+    'ENABLED': false,
+    'GEOLITE2_CITY': '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
+    'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
+    'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
+  },
 };
 
 class Config implements IConfig {
@@ -188,12 +254,16 @@ class Config implements IConfig {
   SYSLOG: IConfig['SYSLOG'];
   STATISTICS: IConfig['STATISTICS'];
   BISQ: IConfig['BISQ'];
+  LIGHTNING: IConfig['LIGHTNING'];
+  LND: IConfig['LND'];
+  CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
+  MAXMIND: IConfig['MAXMIND'];
 
   constructor() {
-    const configs = this.merge(configFile, defaults);
+    const configs = this.merge(configFromFile, defaults);
     this.MEMPOOL = configs.MEMPOOL;
     this.ESPLORA = configs.ESPLORA;
     this.ELECTRUM = configs.ELECTRUM;
@@ -203,9 +273,13 @@ class Config implements IConfig {
     this.SYSLOG = configs.SYSLOG;
     this.STATISTICS = configs.STATISTICS;
     this.BISQ = configs.BISQ;
+    this.LIGHTNING = configs.LIGHTNING;
+    this.LND = configs.LND;
+    this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
+    this.MAXMIND = configs.MAXMIND;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/database.ts

@@ -1,7 +1,7 @@
 import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
-import { PoolOptions } from 'mysql2/typings/mysql';
+import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
 
  class DB {
   constructor() {
@@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql';
     }
   }
 
-  public async query(query, params?) {
+  public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
+    OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
+  {
     this.checkDBFlag();
     const pool = await this.getPool();
     return pool.query(query, params);

backend/src/index.ts

@@ -1,17 +1,14 @@
-import { Express, Request, Response, NextFunction } from 'express';
-import * as express from 'express';
+import express from 'express';
+import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
-import * as cluster from 'cluster';
-import axios from 'axios';
-
+import cluster from 'cluster';
 import DB from './database';
 import config from './config';
-import routes from './routes';
 import blocks from './api/blocks';
 import memPool from './api/mempool';
 import diskCache from './api/disk-cache';
-import statistics from './api/statistics';
+import statistics from './api/statistics/statistics';
 import websocketHandler from './api/websocket-handler';
 import fiatConversion from './api/fiat-conversion';
 import bisq from './api/bisq/bisq';
@@ -27,11 +24,23 @@ import icons from './api/liquid/icons';
 import { Common } from './api/common';
 import poolsUpdater from './tasks/pools-updater';
 import indexer from './indexer';
+import nodesRoutes from './api/explorer/nodes.routes';
+import channelsRoutes from './api/explorer/channels.routes';
+import generalLightningRoutes from './api/explorer/general.routes';
+import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
+import networkSyncService from './tasks/lightning/network-sync.service';
+import statisticsRoutes from './api/statistics/statistics.routes';
+import miningRoutes from './api/mining/mining-routes';
+import bisqRoutes from './api/bisq/bisq.routes';
+import liquidRoutes from './api/liquid/liquid.routes';
+import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
+import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
+import forensicsService from './tasks/lightning/forensics.service';
 
 class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
-  private app: Express;
+  private app: Application;
   private currentBackendRetryInterval = 5;
 
   constructor() {
@@ -42,7 +51,7 @@ class Server {
       return;
     }
 
-    if (cluster.isMaster) {
+    if (cluster.isPrimary) {
       logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
 
       const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
@@ -66,7 +75,7 @@ class Server {
     }
   }
 
-  async startServer(worker = false) {
+  async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
     this.app
@@ -75,8 +84,8 @@ class Server {
         next();
       })
       .use(express.urlencoded({ extended: true }))
-      .use(express.text())
-    ;
+      .use(express.text({ type: ['text/plain', 'application/base64'] }))
+      ;
 
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
@@ -84,7 +93,9 @@ class Server {
     this.setUpWebsocketHandling();
 
     await syncAssets.syncAssets$();
-    diskCache.loadMempoolCache();
+    if (config.MEMPOOL.ENABLED) {
+      diskCache.loadMempoolCache();
+    }
 
     if (config.DATABASE.ENABLED) {
       await DB.checkDbConnection();
@@ -104,7 +115,7 @@ class Server {
       }
     }
 
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
       statistics.startStatistics();
     }
 
@@ -119,7 +130,10 @@ class Server {
     fiatConversion.startService();
 
     this.setUpHttpApiRoutes();
-    this.runMainUpdateLoop();
+
+    if (config.MEMPOOL.ENABLED) {
+      this.runMainUpdateLoop();
+    }
 
     if (config.BISQ.ENABLED) {
       bisq.startBisqService();
@@ -128,6 +142,10 @@ class Server {
       bisqMarkets.startBisqService();
     }
 
+    if (config.LIGHTNING.ENABLED) {
+      this.$runLightningBackend();
+    }
+
     this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
       if (worker) {
         logger.info(`Mempool Server worker #${process.pid} started`);
@@ -137,7 +155,7 @@ class Server {
     });
   }
 
-  async runMainUpdateLoop() {
+  async runMainUpdateLoop(): Promise<void> {
     try {
       try {
         await memPool.$updateMemPoolInfo();
@@ -171,7 +189,20 @@ class Server {
     }
   }
 
-  setUpWebsocketHandling() {
+  async $runLightningBackend(): Promise<void> {
+    try {
+      await fundingTxFetcher.$init();
+      await networkSyncService.$startService();
+      await forensicsService.$startService();
+      await lightningStatsUpdater.$startService();
+    } catch(e) {
+      logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      await Common.sleep$(1000 * 60);
+      this.$runLightningBackend();
+    };
+}
+
+  setUpWebsocketHandling(): void {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);
     }
@@ -185,175 +216,35 @@ class Server {
       });
     }
     websocketHandler.setupConnectionHandling();
-    statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
-    blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
-    memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+    if (config.MEMPOOL.ENABLED) {
+      statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
+      memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+    }
     fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
-
-  setUpHttpApiRoutes() {
-    this.app
-      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', routes.getCpfpInfo)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', routes.getDifficultyChange)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', routes.getMempoolBlocks)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', routes.getBackendInfo)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', routes.getInitData)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', routes.validateAddress)
-      .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
-        try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
-        try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
-            responseType: 'stream', timeout: 10000
-          });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
-        try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
-        try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
-            responseType: 'stream', timeout: 10000
-          });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
-        try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
-        try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
-            responseType: 'stream', timeout: 10000
-          });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-    ;
-
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', routes.$getStatisticsByTime.bind(routes, '2h'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', routes.$getStatisticsByTime.bind(routes, '24h'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', routes.$getStatisticsByTime.bind(routes, '1w'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', routes.$getStatisticsByTime.bind(routes, '1m'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', routes.$getStatisticsByTime.bind(routes, '3m'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', routes.$getStatisticsByTime.bind(routes, '6m'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y'))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y'))
-        ;
+  
+  setUpHttpApiRoutes(): void {
+    bitcoinRoutes.initRoutes(this.app);
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
+      statisticsRoutes.initRoutes(this.app);
     }
-
-    if (Common.indexingEnabled()) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', routes.$getPools)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', routes.$getPoolHistoricalHashrate)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', routes.$getPoolBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', routes.$getPoolBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
-      ;
+    if (Common.indexingEnabled() && config.MEMPOOL.ENABLED) {
+      miningRoutes.initRoutes(this.app);
     }
-
     if (config.BISQ.ENABLED) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', routes.getBisqStats)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', routes.getBisqTransaction)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', routes.getBisqBlock)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', routes.getBisqTip)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', routes.getBisqBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', routes.getBisqAddress)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', routes.getBisqTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', routes.getBisqMarketCurrencies.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', routes.getBisqMarketDepth.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', routes.getBisqMarketHloc.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', routes.getBisqMarketMarkets.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', routes.getBisqMarketOffers.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', routes.getBisqMarketTicker.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', routes.getBisqMarketTrades.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', routes.getBisqMarketVolumes.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', routes.getBisqMarketVolumes7d.bind(routes))
-        ;
+      bisqRoutes.initRoutes(this.app);
     }
-
-    this.app
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock);
-
-    if (config.MEMPOOL.BACKEND !== 'esplora') {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', routes.getMempoolTxIds)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', routes.getRecentMempoolTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction)
-        .post(config.MEMPOOL.API_URL_PREFIX + 'tx', routes.$postTransaction)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', routes.getRawTransaction)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', routes.getTxIdsForBlock)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', routes.getAddress)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix)
-      ;
-    }
-
     if (Common.isLiquid()) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', routes.getAllLiquidIcon)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup)
-      ;
+      liquidRoutes.initRoutes(this.app);
     }
-
-    if (Common.isLiquid() && config.DATABASE.ENABLED) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)
-      ;
+    if (config.LIGHTNING.ENABLED) {
+      generalLightningRoutes.initRoutes(this.app);
+      nodesRoutes.initRoutes(this.app);
+      channelsRoutes.initRoutes(this.app);
     }
   }
 }
 
-const server = new Server();
+((): Server => new Server())();

backend/src/indexer.ts

@@ -1,16 +1,17 @@
 import { Common } from './api/common';
 import blocks from './api/blocks';
 import mempool from './api/mempool';
-import mining from './api/mining';
+import mining from './api/mining/mining';
 import logger from './logger';
 import HashratesRepository from './repositories/HashratesRepository';
+import bitcoinClient from './api/bitcoin/bitcoin-client';
+import priceUpdater from './tasks/price-updater';
+import PricesRepository from './repositories/PricesRepository';
 
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
-
-  constructor() {
-  }
+  tasksRunning: string[] = [];
 
   public reindex() {
     if (Common.indexingEnabled()) {
@@ -18,6 +19,28 @@ class Indexer {
     }
   }
 
+  public async runSingleTask(task: 'blocksPrices') {
+    if (!Common.indexingEnabled()) {
+      return;
+    }
+
+    if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      const lastestPriceId = await PricesRepository.$getLatestPriceId();
+      if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        setTimeout(() => {
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.runSingleTask('blocksPrices');
+        }, 10000);
+      } else {
+        logger.debug(`Blocks prices indexer will run now`)
+        await mining.$indexBlockPrices();
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+      }
+    }
+  }
+
   public async $run() {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
@@ -25,20 +48,49 @@ class Indexer {
       return;
     }
 
+    // Do not attempt to index anything unless Bitcoin Core is fully synced
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    if (blockchainInfo.blocks !== blockchainInfo.headers) {
+      return;
+    }
+
     this.runIndexer = false;
     this.indexerRunning = true;
 
+    logger.debug(`Running mining indexer`);
+
     try {
-      await blocks.$generateBlockDatabase();
-      await this.$resetHashratesIndexingState();
+      await priceUpdater.$run();
+
+      const chainValid = await blocks.$generateBlockDatabase();
+      if (chainValid === false) {
+        // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
+        setTimeout(() => this.reindex(), 10000);
+        this.indexerRunning = false;
+        return;
+      }
+
+      this.runSingleTask('blocksPrices');
+      await mining.$indexDifficultyAdjustments();
+      await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();
+      await blocks.$generateBlocksSummariesDatabase();
+      await blocks.$generateCPFPDatabase();
     } catch (e) {
-      this.reindex();
-      logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
+      this.indexerRunning = false;
+      logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
+      setTimeout(() => this.reindex(), 10000);
+      this.indexerRunning = false;
+      return;
     }
 
     this.indexerRunning = false;
+
+    const runEvery = 1000 * 3600; // 1 hour
+    logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
+    setTimeout(() => this.reindex(), runEvery);
   }
 
   async $resetHashratesIndexingState() {
@@ -47,6 +99,7 @@ class Indexer {
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
     } catch (e) {
       logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
     }
   }
 }

backend/src/logger.ts

@@ -73,6 +73,9 @@ class Logger {
   }
 
   private getNetwork(): string {
+    if (config.LIGHTNING.ENABLED) {
+      return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; 
+    }
     if (config.BISQ.ENABLED) {
       return 'bisq';
     }

backend/src/mempool.interfaces.ts

@@ -1,4 +1,5 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { HeapNode } from "./utils/pairing-heap";
 
 export interface PoolTag {
   id: number; // mysql row id
@@ -22,6 +23,21 @@ export interface PoolStats extends PoolInfo {
   emptyBlocks: number;
 }
 
+export interface BlockAudit {
+  time: number,
+  height: number,
+  hash: string,
+  missingTxs: string[],
+  freshTxs: string[],
+  addedTxs: string[],
+  matchRate: number,
+}
+
+export interface AuditScore {
+  hash: string,
+  matchRate?: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;
@@ -56,17 +72,46 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   firstSeen?: number;
   effectiveFeePerVsize: number;
   ancestors?: Ancestor[];
+  descendants?: Ancestor[];
   bestDescendant?: BestDescendant | null;
   cpfpChecked?: boolean;
   deleteAfter?: number;
 }
 
-interface Ancestor {
+export interface AuditTransaction {
+  txid: string;
+  fee: number;
+  size: number;
+  weight: number;
+  feePerVsize: number;
+  vin: IEsploraApi.Vin[];
+  relativesSet: boolean;
+  ancestorMap: Map<string, AuditTransaction>;
+  children: Set<AuditTransaction>;
+  ancestorFee: number;
+  ancestorWeight: number;
+  score: number;
+  used: boolean;
+  modified: boolean;
+  modifiedNode: HeapNode<AuditTransaction>;
+}
+
+export interface Ancestor {
   txid: string;
   weight: number;
   fee: number;
 }
 
+export interface TransactionSet {
+  fee: number;
+  weight: number;
+  score: number;
+  children?: Set<string>;
+  available?: boolean;
+  modified?: boolean;
+  modifiedNode?: HeapNode<string>;
+}
+
 interface BestDescendant {
   txid: string;
   weight: number;
@@ -75,7 +120,9 @@ interface BestDescendant {
 
 export interface CpfpInfo {
   ancestors: Ancestor[];
-  bestDescendant: BestDescendant | null;
+  bestDescendant?: BestDescendant | null;
+  descendants?: Ancestor[];
+  effectiveFeePerVsize?: number;
 }
 
 export interface TransactionStripped {
@@ -100,12 +147,23 @@ export interface BlockExtension {
   avgFee?: number;
   avgFeeRate?: number;
   coinbaseRaw?: string;
+  usd?: number | null;
 }
 
 export interface BlockExtended extends IEsploraApi.Block {
   extras: BlockExtension;
 }
 
+export interface BlockSummary {
+  id: string;
+  transactions: TransactionStripped[];
+}
+
+export interface BlockPrice {
+  height: number;
+  priceId: number;
+}
+
 export interface TransactionMinerInfo {
   vin: VinStrippedToScriptsig[];
   vout: VoutStrippedToScriptPubkey[];
@@ -219,8 +277,53 @@ export interface IDifficultyAdjustment {
   timeOffset: number;
 }
 
+export interface IndexedDifficultyAdjustment {
+  time: number; // UNIX timestamp
+  height: number; // Block height
+  difficulty: number;
+  adjustment: number;
+}
+
 export interface RewardStats {
   totalReward: number;
   totalFee: number;
   totalTx: number;
 }
+
+export interface ITopNodesPerChannels {
+  publicKey: string,
+  alias: string,
+  channels?: number,
+  capacity: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface ITopNodesPerCapacity {
+  publicKey: string,
+  alias: string,
+  capacity: number,
+  channels?: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface INodesRanking {
+  topByCapacity: ITopNodesPerCapacity[];
+  topByChannels: ITopNodesPerChannels[];
+}
+
+export interface IOldestNodes {
+  publicKey: string,
+  alias: string,
+  firstSeen: number,
+  channels?: number,
+  capacity: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}

backend/src/repositories/BlocksAuditsRepository.ts

@@ -0,0 +1,110 @@
+import blocks from '../api/blocks';
+import DB from '../database';
+import logger from '../logger';
+import { BlockAudit, AuditScore } from '../mempool.interfaces';
+
+class BlocksAuditRepositories {
+  public async $saveAudit(audit: BlockAudit): Promise<void> {
+    try {
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, match_rate)
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), audit.matchRate]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save block audit into db. Reason: ` + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+
+  public async $getBlockPredictionsHistory(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
+
+      if (interval !== null) {
+        query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${div} ORDER BY height`;
+
+      const [rows] = await DB.query(query);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getPredictionsCount(): Promise<number> {
+    try {
+      const [rows] = await DB.query(`SELECT count(hash) as count FROM blocks_audits`);
+      return rows[0].count;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAudit(hash: string): Promise<any> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
+        blocks.weight, blocks.tx_count,
+        transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, match_rate as matchRate
+        FROM blocks_audits
+        JOIN blocks ON blocks.hash = blocks_audits.hash
+        JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
+        WHERE blocks_audits.hash = "${hash}"
+      `);
+      
+      if (rows.length) {
+        rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
+        rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
+        rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
+        rows[0].transactions = JSON.parse(rows[0].transactions);
+        rows[0].template = JSON.parse(rows[0].template);
+
+        if (rows[0].transactions.length) {
+          return rows[0];
+        }
+      }
+      return null;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.hash = "${hash}"
+      `);
+      return rows[0];
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise<AuditScore[]> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.height BETWEEN ? AND ?
+      `, [minHeight, maxHeight]);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new BlocksAuditRepositories();
+

backend/src/repositories/BlocksRepository.ts

@@ -1,4 +1,4 @@
-import { BlockExtended } from '../mempool.interfaces';
+import { BlockExtended, BlockPrice } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
@@ -6,6 +6,8 @@ import { prepareBlock } from '../utils/blocks-utils';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
+import BlocksSummariesRepository from './BlocksSummariesRepository';
+import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
 
 class BlocksRepository {
   /**
@@ -254,7 +256,7 @@ class BlocksRepository {
 
     const params: any[] = [];
     let query = ` SELECT
-      height,
+      blocks.height,
       hash as id,
       UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
       size,
@@ -306,7 +308,7 @@ class BlocksRepository {
   public async $getBlockByHeight(height: number): Promise<object | null> {
     try {
       const [rows]: any[] = await DB.query(`SELECT
-        height,
+        blocks.height,
         hash,
         hash as id,
         UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
@@ -334,7 +336,7 @@ class BlocksRepository {
         avg_fee_rate
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
-        WHERE height = ${height};
+        WHERE blocks.height = ${height}
       `);
 
       if (rows.length <= 0) {
@@ -355,15 +357,15 @@ class BlocksRepository {
   public async $getBlockByHash(hash: string): Promise<object | null> {
     try {
       const query = `
-        SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
+        SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
         pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
         pools.addresses as pool_addresses, pools.regexes as pool_regexes,
         previous_block_hash as previousblockhash
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
-        WHERE hash = '${hash}';
+        WHERE hash = ?;
       `;
-      const [rows]: any[] = await DB.query(query);
+      const [rows]: any[] = await DB.query(query, [hash]);
 
       if (rows.length <= 0) {
         return null;
@@ -380,51 +382,55 @@ class BlocksRepository {
   /**
    * Return blocks difficulty
    */
-  public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
-    interval = Common.getSqlInterval(interval);
-
-    // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
-    // Basically, using temporary user defined fields, we are able to extract all
-    // difficulty adjustments from the blocks tables.
-    // This allow use to avoid indexing it in another table.
-    let query = `
-      SELECT
-      *
-      FROM
-      (
-        SELECT
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height,
-        IF(@prevStatus = YT.difficulty, @rn := @rn + 1,
-          IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1)
-        ) AS rn
-        FROM blocks YT
-        CROSS JOIN
-        (
-          SELECT @prevStatus := -1, @rn := 1
-        ) AS var
-    `;
-
-    if (interval) {
-      query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    query += `
-        ORDER BY YT.height
-      ) AS t
-      WHERE t.rn = 1
-      ORDER BY t.height
-    `;
-
+  public async $getBlocksDifficulty(): Promise<object[]> {
     try {
-      const [rows]: any[] = await DB.query(query);
-
-      for (const row of rows) {
-        delete row['rn'];
-      }
-
+      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
       return rows;
     } catch (e) {
-      logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the first block at or directly after a given timestamp
+   * @param timestamp number unix time in seconds
+   * @returns The height and timestamp of a block (timestamp might vary from given timestamp)
+   */
+  public async $getBlockHeightFromTimestamp(
+    timestamp: number,
+  ): Promise<{ height: number; hash: string; timestamp: number }> {
+    try {
+      // Get first block at or after the given timestamp
+      const query = `SELECT height, hash, blockTimestamp as timestamp FROM blocks
+        WHERE blockTimestamp <= FROM_UNIXTIME(?)
+        ORDER BY blockTimestamp DESC
+        LIMIT 1`;
+      const params = [timestamp];
+      const [rows]: any[][] = await DB.query(query, params);
+      if (rows.length === 0) {
+        throw new Error(`No block was found before timestamp ${timestamp}`);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err(
+        'Cannot get block height from timestamp from the db. Reason: ' +
+          (e instanceof Error ? e.message : e),
+      );
+      throw e;
+    }
+  }
+
+  /**
+   * Return blocks height
+   */
+   public async $getBlocksHeightsAndTimestamp(): Promise<object[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT height, blockTimestamp as timestamp FROM blocks`);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -451,26 +457,6 @@ class BlocksRepository {
     }
   }
 
-  /*
-   * Check if the last 10 blocks chain is valid
-   */
-  public async $validateRecentBlocks(): Promise<boolean> {
-    try {
-      const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
-
-      for (let i = 0; i < lastBlocks.length - 1; ++i) {
-        if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
-          logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
-          return false;
-        }
-      }
-
-      return true;
-    } catch (e) {
-      return true; // Don't do anything if there is a db error
-    }
-  }
-
   /**
    * Check if the chain of block hash is valid and delete data from the stale branch if needed
    */
@@ -493,15 +479,17 @@ class BlocksRepository {
         }
 
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
-          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
+          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
+          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
+          await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
           return false;
         }
         ++idx;
       }
 
-      logger.info(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`);
+      logger.debug(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`);
       return true;
     } catch (e) {
       logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e));
@@ -528,10 +516,14 @@ class BlocksRepository {
   public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
     try {
       let query = `SELECT
-        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(blocks.height) as INT) as avgHeight,
         CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(fees) as INT) as avgFees
-        FROM blocks`;
+        CAST(AVG(fees) as INT) as avgFees,
+        prices.USD
+        FROM blocks
+        JOIN blocks_prices on blocks_prices.height = blocks.height
+        JOIN prices on prices.id = blocks_prices.price_id
+      `;
 
       if (interval !== null) {
         query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
@@ -553,10 +545,14 @@ class BlocksRepository {
   public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
     try {
       let query = `SELECT
-        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(blocks.height) as INT) as avgHeight,
         CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(reward) as INT) as avgRewards
-        FROM blocks`;
+        CAST(AVG(reward) as INT) as avgRewards,
+        prices.USD
+        FROM blocks
+        JOIN blocks_prices on blocks_prices.height = blocks.height
+        JOIN prices on prices.id = blocks_prices.price_id
+      `;
 
       if (interval !== null) {
         query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
@@ -652,6 +648,94 @@ class BlocksRepository {
       throw e;
     }
   }
+
+  /**
+   * Get a list of blocks that have been indexed
+   */
+  public async $getIndexedBlocks(): Promise<any[]> {
+    try {
+      const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get a list of blocks that have not had CPFP data indexed
+   */
+   public async $getCPFPUnindexedBlocks(): Promise<any[]> {
+    try {
+      const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $setCPFPIndexed(hash: string): Promise<void> {
+    await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]);
+  }
+
+  /**
+   * Return the oldest block  from a consecutive chain of block from the most recent one
+   */
+  public async $getOldestConsecutiveBlock(): Promise<any> {
+    try {
+      const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`);
+      for (let i = 0; i < rows.length - 1; ++i) {
+        if (rows[i].height - rows[i + 1].height > 1) {
+          return rows[i];
+        }
+      }
+      return rows[rows.length - 1];
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get all blocks which have not be linked to a price yet
+   */
+   public async $getBlocksWithoutPrice(): Promise<object[]> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.height
+        FROM blocks
+        LEFT JOIN blocks_prices ON blocks.height = blocks_prices.height
+        WHERE blocks_prices.height IS NULL
+        ORDER BY blocks.height
+      `);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save block price by batch
+   */
+   public async $saveBlockPrices(blockPrices: BlockPrice[]): Promise<void> {
+    try {
+      let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
+      for (const price of blockPrices) {
+        query += ` (${price.height}, ${price.priceId}),`
+      }
+      query = query.slice(0, -1);
+      await DB.query(query);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] because it has already been indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] into db. Reason: ` + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
 }
 
 export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -0,0 +1,86 @@
+import DB from '../database';
+import logger from '../logger';
+import { BlockSummary } from '../mempool.interfaces';
+
+class BlocksSummariesRepository {
+  public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
+    try {
+      const [summary]: any[] = await DB.query(`SELECT * from blocks_summaries WHERE id = ?`, [id]);
+      if (summary.length > 0) {
+        summary[0].transactions = JSON.parse(summary[0].transactions);
+        return summary[0];
+      }
+    } catch (e) {
+      logger.err(`Cannot get block summary for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return undefined;
+  }
+
+  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
+    const blockId = params.mined?.id;
+    try {
+      const transactions = JSON.stringify(params.mined?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          transactions = ?
+      `, [params.height, blockId, transactions, '[]', transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $saveTemplate(params: { height: number, template: BlockSummary}) {
+    const blockId = params.template?.id;
+    try {
+      const transactions = JSON.stringify(params.template?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          template = ?
+      `, [params.height, blockId, '[]', transactions, transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getIndexedSummariesId(): Promise<string[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);
+      return rows.map(row => row.id);
+    } catch (e) {
+      logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return [];
+  }
+
+  /**
+   * Delete blocks from the database from blockHeight
+   */
+  public async $deleteBlocksFrom(blockHeight: number) {
+    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new BlocksSummariesRepository();
+

backend/src/repositories/CpfpRepository.ts

@@ -0,0 +1,43 @@
+import DB from '../database';
+import logger from '../logger';
+import { Ancestor } from '../mempool.interfaces';
+
+class CpfpRepository {
+  public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
+    try {
+      const txsJson = JSON.stringify(txs);
+      await DB.query(
+        `
+          INSERT INTO cpfp_clusters(root, height, txs, fee_rate)
+          VALUE (?, ?, ?, ?)
+          ON DUPLICATE KEY UPDATE
+            height = ?,
+            txs = ?,
+            fee_rate = ?
+        `,
+        [txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $deleteClustersFrom(height: number): Promise<void> {
+    logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
+    try {
+      await DB.query(
+        `
+          DELETE from cpfp_clusters
+          WHERE height >= ?
+        `,
+        [height]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new CpfpRepository();

backend/src/repositories/DifficultyAdjustmentsRepository.ts

@@ -0,0 +1,124 @@
+import { Common } from '../api/common';
+import config from '../config';
+import DB from '../database';
+import logger from '../logger';
+import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
+
+class DifficultyAdjustmentsRepository {
+  public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise<void> {
+    if (adjustment.height === 1) {
+      return;
+    }
+
+    try {
+      const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`;
+      const params: any[] = [
+        adjustment.time,
+        adjustment.height,
+        adjustment.difficulty,
+        adjustment.adjustment,
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT 
+      CAST(AVG(UNIX_TIMESTAMP(time)) as INT) as time,
+      CAST(AVG(height) AS INT) as height,
+      CAST(AVG(difficulty) as DOUBLE) as difficulty,
+      CAST(AVG(adjustment) as DOUBLE) as adjustment
+      FROM difficulty_adjustments`;
+
+    if (interval) {
+      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${86400}`;
+
+    if (descOrder === true) {
+      query += ` ORDER BY height DESC`;
+    } else {
+      query += ` ORDER BY height`;
+    }
+
+    try {
+      const [rows] = await DB.query(query);
+      return rows as IndexedDifficultyAdjustment[];
+    } catch (e) {
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getRawAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT 
+      UNIX_TIMESTAMP(time) as time,
+      height as height,
+      difficulty as difficulty,
+      adjustment as adjustment
+      FROM difficulty_adjustments`;
+
+    if (interval) {
+      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    if (descOrder === true) {
+      query += ` ORDER BY height DESC`;
+    } else {
+      query += ` ORDER BY height`;
+    }
+
+    try {
+      const [rows] = await DB.query(query);
+      return rows as IndexedDifficultyAdjustment[];
+    } catch (e) {
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getAdjustmentsHeights(): Promise<number[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
+      return rows.map(block => block.height);
+    } catch (e: any) {
+      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
+    try {
+      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
+    } catch (e: any) {
+      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteLastAdjustment(): Promise<void> {
+    try {
+      logger.info(`Delete last difficulty adjustment from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
+    } catch (e: any) {
+      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+}
+
+export default new DifficultyAdjustmentsRepository();
+

backend/src/repositories/HashratesRepository.ts

@@ -29,10 +29,12 @@ class HashratesRepository {
     }
   }
 
-  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
+  public async $getRawNetworkDailyHashrate(interval: string | null): Promise<any[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
+    let query = `SELECT
+      UNIX_TIMESTAMP(hashrate_timestamp) as timestamp,
+      avg_hashrate as avgHashrate
       FROM hashrates`;
 
     if (interval) {
@@ -53,6 +55,33 @@ class HashratesRepository {
     }
   }
 
+  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT
+      CAST(AVG(UNIX_TIMESTAMP(hashrate_timestamp)) as INT) as timestamp,
+      CAST(AVG(avg_hashrate) as DOUBLE) as avgHashrate
+      FROM hashrates`;
+
+    if (interval) {
+      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+        AND hashrates.type = 'daily'`;
+    } else {
+      query += ` WHERE hashrates.type = 'daily'`;
+    }
+
+    query += ` GROUP BY UNIX_TIMESTAMP(hashrate_timestamp) DIV ${86400}`;
+    query += ` ORDER by hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
     const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp
       FROM hashrates
@@ -75,6 +104,9 @@ class HashratesRepository {
     interval = Common.getSqlInterval(interval);
 
     const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
+    if (topPoolsId.length === 0) {
+      return [];
+    }
 
     let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
       FROM hashrates

backend/src/repositories/NodeRecordsRepository.ts

@@ -0,0 +1,67 @@
+import { ResultSetHeader, RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeRecord {
+  publicKey: string; // node public key
+  type: number; // TLV extension record type
+  payload: string; // base64 record payload
+}
+
+class NodesRecordsRepository {
+  public async $saveRecord(record: NodeRecord): Promise<void> {
+    try {
+      const payloadBytes = Buffer.from(record.payload, 'base64');
+      await DB.query(`
+        INSERT INTO nodes_records(public_key, type, payload)
+        VALUE (?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          payload = ?
+      `, [record.publicKey, record.type, payloadBytes, payloadBytes]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node record (${[record.publicKey, record.type, record.payload]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes records
+      }
+    }
+   }
+
+  public async $getRecordTypes(publicKey: string): Promise<any> {
+    try {
+      const query = `
+        SELECT type FROM nodes_records
+        WHERE public_key = ?
+      `;
+      const [rows] = await DB.query<RowDataPacket[][]>(query, [publicKey]);
+      return rows.map(row => row['type']);
+    } catch (e) {
+      logger.err(`Cannot retrieve custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise<number> {
+    try {
+      let query;
+      if (recordTypes.length) {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+          AND type NOT IN (${recordTypes.map(type => `${type}`).join(',')})
+        `;
+      } else {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+        `;
+      }
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+  }
+}
+
+export default new NodesRecordsRepository();

backend/src/repositories/NodesSocketsRepository.ts

@@ -0,0 +1,45 @@
+import { ResultSetHeader } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeSocket {
+  publicKey: string;
+  network: string | null;
+  addr: string;
+}
+
+class NodesSocketsRepository {
+  public async $saveSocket(socket: NodeSocket): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO nodes_sockets(public_key, socket, type)
+        VALUE (?, ?, ?)
+      `, [socket.publicKey, socket.addr, socket.network]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes sockets
+      }
+    }
+   }
+
+   public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
+    if (addresses.length === 0) {
+      return 0;
+    }
+    try {
+      const query = `
+        DELETE FROM nodes_sockets
+        WHERE public_key = ?
+        AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
+      `;
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+   }
+}
+
+export default new NodesSocketsRepository();

backend/src/repositories/PricesRepository.ts

@@ -0,0 +1,52 @@
+import DB from '../database';
+import logger from '../logger';
+import { Prices } from '../tasks/price-updater';
+
+class PricesRepository {
+  public async $savePrices(time: number, prices: Prices): Promise<void> {
+    if (prices.USD === -1) {
+      // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
+      // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
+      return;
+    }
+
+    try {
+      await DB.query(`
+        INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
+        VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
+        [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestPriceTime(): Promise<number> {
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].time : 0;
+  }
+
+  public async $getLatestPriceId(): Promise<number | null> {
+    const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].id : null;
+  }
+
+  public async $getLatestPriceTime(): Promise<number> {
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].time : 0;
+  }
+
+  public async $getPricesTimes(): Promise<number[]> {
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
+    return times.map(time => time.time);
+  }
+
+  public async $getPricesTimesAndId(): Promise<number[]> {
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
+    return times;
+  }
+}
+
+export default new PricesRepository();
+

backend/src/repositories/RatesRepository.ts

@@ -1,21 +0,0 @@
-import DB from '../database';
-import logger from '../logger';
-import { IConversionRates } from '../mempool.interfaces';
-
-class RatesRepository {
-  public async $saveRate(height: number, rates: IConversionRates) {
-    try {
-      await DB.query(`INSERT INTO rates(height, bisq_rates) VALUE (?, ?)`, [height, JSON.stringify(rates)]);
-    } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Rate already exists for block ${height}, ignoring`);
-      } else {
-        logger.err(`Cannot save exchange rate into db for block ${height} Reason: ` + (e instanceof Error ? e.message : e));
-        throw e;
-      }
-    }
-  }
-}
-
-export default new RatesRepository();
-

backend/src/repositories/TransactionRepository.ts

@@ -0,0 +1,77 @@
+import DB from '../database';
+import logger from '../logger';
+import { Ancestor, CpfpInfo } from '../mempool.interfaces';
+
+interface CpfpSummary {
+  txid: string;
+  cluster: string;
+  root: string;
+  txs: Ancestor[];
+  height: number;
+  fee_rate: number;
+}
+
+class TransactionRepository {
+  public async $setCluster(txid: string, cluster: string): Promise<void> {
+    try {
+      await DB.query(
+        `
+          INSERT INTO transactions
+          (
+            txid,
+            cluster
+          )
+          VALUE (?, ?)
+          ON DUPLICATE KEY UPDATE
+            cluster = ?
+        ;`,
+        [txid, cluster, cluster]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
+    try {
+      let query = `
+        SELECT *
+        FROM transactions
+        LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster
+        WHERE transactions.txid = ?
+      `;
+      const [rows]: any = await DB.query(query, [txid]);
+      if (rows.length) {
+        rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[];
+        return this.convertCpfp(rows[0]);
+      }
+    } catch (e) {
+      logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private convertCpfp(cpfp: CpfpSummary): CpfpInfo {
+    const descendants: Ancestor[] = [];
+    const ancestors: Ancestor[] = [];
+    let matched = false;
+    for (const tx of cpfp.txs) {
+      if (tx.txid === cpfp.txid) {
+        matched = true;
+      } else if (!matched) {
+        descendants.push(tx);
+      } else {
+        ancestors.push(tx);
+      }
+    }
+    return {
+      descendants,
+      ancestors,
+      effectiveFeePerVsize: cpfp.fee_rate
+    };
+  }
+}
+
+export default new TransactionRepository();
+

backend/src/tasks/lightning/forensics.service.ts

@@ -0,0 +1,457 @@
+import DB from '../../database';
+import logger from '../../logger';
+import channelsApi from '../../api/explorer/channels.api';
+import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
+import config from '../../config';
+import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
+import { Common } from '../../api/common';
+import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+
+const tempCacheSize = 10000;
+
+class ForensicsService {
+  loggerTimer = 0;
+  closedChannelsScanBlock = 0;
+  txCache: { [txid: string]: IEsploraApi.Transaction } = {};
+  tempCached: string[] = [];
+
+  constructor() {}
+
+  public async $startService(): Promise<void> {
+    logger.info('Starting lightning network forensics service');
+
+    this.loggerTimer = new Date().getTime() / 1000;
+
+    await this.$runTasks();
+  }
+
+  private async $runTasks(): Promise<void> {
+    try {
+      logger.info(`Running forensics scans`);
+
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        await this.$runClosedChannelsForensics(false);
+        await this.$runOpenedChannelsForensics();
+      }
+
+    } catch (e) {
+      logger.err('ForensicsService.$runTasks() error: ' + (e instanceof Error ? e.message : e));
+    }
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.FORENSICS_INTERVAL);
+  }
+
+  /*
+    1. Mutually closed
+    2. Forced closed
+    3. Forced closed with penalty
+
+    ┌────────────────────────────────────┐       ┌────────────────────────────┐
+    │ outputs contain revocation script? ├──yes──► force close w/ penalty = 3 │
+    └──────────────┬─────────────────────┘       └────────────────────────────┘
+                   no
+    ┌──────────────▼──────────────────────────┐
+    │ outputs contain other lightning script? ├──┐
+    └──────────────┬──────────────────────────┘  │
+                   no                           yes
+    ┌──────────────▼─────────────┐               │
+    │ sequence starts with 0x80  │      ┌────────▼────────┐
+    │           and              ├──────► force close = 2 │
+    │ locktime starts with 0x20? │      └─────────────────┘
+    └──────────────┬─────────────┘
+                   no
+         ┌─────────▼────────┐
+         │ mutual close = 1 │
+         └──────────────────┘
+  */
+
+  public async $runClosedChannelsForensics(onlyNewChannels: boolean = false): Promise<void> {
+    if (config.MEMPOOL.BACKEND !== 'esplora') {
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      logger.info(`Started running closed channel forensics...`);
+      let channels;
+      if (onlyNewChannels) {
+        channels = await channelsApi.$getClosedChannelsWithoutReason();
+      } else {
+        channels = await channelsApi.$getUnresolvedClosedChannels();
+      }
+
+      for (const channel of channels) {
+        let reason = 0;
+        let resolvedForceClose = false;
+        // Only Esplora backend can retrieve spent transaction outputs
+        const cached: string[] = [];
+        try {
+          let outspends: IEsploraApi.Outspend[] | undefined;
+          try {
+            outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
+            await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+          } catch (e) {
+            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+            continue;
+          }
+          const lightningScriptReasons: number[] = [];
+          for (const outspend of outspends) {
+            if (outspend.spent && outspend.txid) {
+              let spendingTx = await this.fetchTransaction(outspend.txid);
+              if (!spendingTx) {
+                continue;
+              }
+              cached.push(spendingTx.txid);
+              const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              lightningScriptReasons.push(lightningScript);
+            }
+          }
+          const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
+          if (filteredReasons.length) {
+            if (filteredReasons.some((r) => r === 2 || r === 4)) {
+              reason = 3;
+            } else {
+              reason = 2;
+              resolvedForceClose = true;
+            }
+          } else {
+            /*
+              We can detect a commitment transaction (force close) by reading Sequence and Locktime
+              https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
+            */
+            let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true);
+            if (!closingTx) {
+              continue;
+            }
+            cached.push(closingTx.txid);
+            const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
+            const locktimeHex: string = closingTx.locktime.toString(16);
+            if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
+              reason = 2; // Here we can't be sure if it's a penalty or not
+            } else {
+              reason = 1;
+            }
+          }
+          if (reason) {
+            logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
+            await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+            if (reason === 2 && resolvedForceClose) {
+              await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]);
+            }
+            if (reason !== 2 || resolvedForceClose) {
+              cached.forEach(txid => {
+                delete this.txCache[txid];
+              });
+            }
+          }
+        } catch (e) {
+          logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Closed channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private findLightningScript(vin: IEsploraApi.Vin): number {
+    const topElement = vin.witness?.length > 2 ? vin.witness[vin.witness.length - 2] : null;
+      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
+        if (topElement === '01') {
+          // top element is '01' to get in the revocation path
+          // 'Revoked Lightning Force Close';
+          // Penalty force closed
+          return 2;
+        } else {
+          // top element is '', this is a delayed to_local output
+          // 'Lightning Force Close';
+          return 3;
+        }
+      } else if (
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
+      ) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
+        if (topElement?.length === 66) {
+          // top element is a public key
+          // 'Revoked Lightning HTLC'; Penalty force closed
+          return 4;
+        } else if (topElement) {
+          // top element is a preimage
+          // 'Lightning HTLC';
+          return 5;
+        } else {
+          // top element is '' to get in the expiry of the script
+          // 'Expired Lightning HTLC';
+          return 6;
+        }
+      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
+        if (topElement) {
+          // top element is a signature
+          // 'Lightning Anchor';
+          return 7;
+        } else {
+          // top element is '', it has been swept after 16 blocks
+          // 'Swept Lightning Anchor';
+          return 8;
+        }
+      }
+      return 1;
+  }
+
+  // If a channel open tx spends funds from a another channel transaction,
+  // we can attribute that output to a specific counterparty
+  private async $runOpenedChannelsForensics(): Promise<void> {
+    const runTimer = Date.now();
+    let progress = 0;
+
+    try {
+      logger.info(`Started running open channel forensics...`);
+      const channels = await channelsApi.$getChannelsWithoutSourceChecked();
+
+      for (const openChannel of channels) {
+        let openTx = await this.fetchTransaction(openChannel.transaction_id, true);
+        if (!openTx) {
+          continue;
+        }
+        for (const input of openTx.vin) {
+          const closeChannel = await channelsApi.$getChannelByClosingId(input.txid);
+          if (closeChannel) {
+            // this input directly spends a channel close output
+            await this.$attributeChannelBalances(closeChannel, openChannel, input);
+          } else {
+            const prevOpenChannels = await channelsApi.$getChannelsByOpeningId(input.txid);
+            if (prevOpenChannels?.length) {
+              // this input spends a channel open change output
+              for (const prevOpenChannel of prevOpenChannels) {
+                await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true);
+              }
+            } else {
+              // check if this input spends any swept channel close outputs
+              await this.$attributeSweptChannelCloses(openChannel, input);
+            }
+          }
+        }
+        // calculate how much of the total input value is attributable to the channel open output
+        openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee);
+        // save changes to the opening channel, and mark it as checked
+        if (openTx?.vin?.length === 1) {
+          openChannel.single_funded = true;
+        }
+        if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) {
+          await channelsApi.$updateOpeningInfo(openChannel);
+        }
+        await channelsApi.$markChannelSourceChecked(openChannel.id);
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+          this.truncateTempCache();
+        }
+        if (Date.now() - runTimer > (config.LIGHTNING.FORENSICS_INTERVAL * 1000)) {
+          break;
+        }
+      }
+
+      logger.info(`Open channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    } finally {
+      this.clearTempCache();
+    }
+  }
+
+  // Check if a channel open tx input spends the result of a swept channel close output
+  private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
+    let sweepTx = await this.fetchTransaction(input.txid, true);
+    if (!sweepTx) {
+      logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
+      return;
+    }
+    const openContribution = sweepTx.vout[input.vout].value;
+    for (const sweepInput of sweepTx.vin) {
+      const lnScriptType = this.findLightningScript(sweepInput);
+      if (lnScriptType > 1) {
+        const closeChannel = await channelsApi.$getChannelByClosingId(sweepInput.txid);
+        if (closeChannel) {
+          const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null);
+          await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator);
+        }
+      }
+    }
+  }
+
+  private async $attributeChannelBalances(
+    prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
+    initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false
+  ): Promise<void> {
+    // figure out which node controls the input/output
+    let openSide;
+    let prevLocal;
+    let prevRemote;
+    let matched = false;
+    let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart
+    if (openChannel.node1_public_key === prevChannel.node1_public_key) {
+      openSide = 1;
+      prevLocal = 1;
+      prevRemote = 2;
+      matched = true;
+    } else if (openChannel.node1_public_key === prevChannel.node2_public_key) {
+      openSide = 1;
+      prevLocal = 2;
+      prevRemote = 1;
+      matched = true;
+    }
+    if (openChannel.node2_public_key === prevChannel.node1_public_key) {
+      openSide = 2;
+      prevLocal = 1;
+      prevRemote = 2;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    } else if (openChannel.node2_public_key === prevChannel.node2_public_key) {
+      openSide = 2;
+      prevLocal = 2;
+      prevRemote = 1;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    }
+
+    if (matched && !ambiguous) {
+      // fetch closing channel transaction and perform forensics on the outputs
+      let prevChannelTx = await this.fetchTransaction(input.txid, true);
+      let outspends: IEsploraApi.Outspend[] | undefined;
+      try {
+        outspends = await bitcoinApi.$getOutspends(input.txid);
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+      } catch (e) {
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+      }
+      if (!outspends || !prevChannelTx) {
+        return;
+      }
+      if (!linkedOpenings) {
+        if (!prevChannel.outputs || !prevChannel.outputs.length) {
+          prevChannel.outputs = prevChannelTx.vout.map(vout => {
+            return {
+              type: 0,
+              value: vout.value,
+            };
+          });
+        }
+        for (let i = 0; i < outspends?.length; i++) {
+          const outspend = outspends[i];
+          const output = prevChannel.outputs[i];
+          if (outspend.spent && outspend.txid) {
+            try {
+              const spendingTx = await this.fetchTransaction(outspend.txid, true);
+              if (spendingTx) {
+                output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              }
+            } catch (e) {
+              logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
+            }
+          } else {
+            output.type = 0;
+          }
+        }
+
+        // attribute outputs to each counterparty, and sum up total known balances
+        prevChannel.outputs[input.vout].node = prevLocal;
+        const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0;
+        const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type);
+        const mutualClose = ((prevChannel.status === 2 || prevChannel.status === 'closed') && prevChannel.closing_reason === 1);
+        let localClosingBalance = 0;
+        let remoteClosingBalance = 0;
+        for (const output of prevChannel.outputs) {
+          if (isPenalty) {
+            // penalty close, so local node takes everything
+            localClosingBalance += output.value;
+          } else if (output.node) {
+            // this output determinstically linked to one of the counterparties
+            if (output.node === prevLocal) {
+              localClosingBalance += output.value;
+            } else {
+              remoteClosingBalance += output.value;
+            }
+          } else if (normalOutput && (output.type === 1 || output.type === 3 || (mutualClose && prevChannel.outputs.length === 2))) {
+            // local node had one main output, therefore remote node takes the other
+            remoteClosingBalance += output.value;
+          }
+        }
+        prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance;
+        prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance;
+        prevChannel.closing_fee = prevChannelTx.fee;
+
+        if (initiator && !linkedOpenings) {
+          const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal;
+          prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`];
+        }
+  
+        // save changes to the closing channel
+        await channelsApi.$updateClosingInfo(prevChannel);
+      } else {
+        if (prevChannelTx.vin.length <= 1) {
+          prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity;
+          prevChannel.single_funded = true;
+          prevChannel.funding_ratio = 1;
+          // save changes to the closing channel
+          await channelsApi.$updateOpeningInfo(prevChannel);
+        }
+      }
+      openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0);
+    }
+  }
+
+  async fetchTransaction(txid: string, temp: boolean = false): Promise<IEsploraApi.Transaction | null> {
+    let tx = this.txCache[txid];
+    if (!tx) {
+      try {
+        tx = await bitcoinApi.$getRawTransaction(txid);
+        this.txCache[txid] = tx;
+        if (temp) {
+          this.tempCached.push(txid);
+        }
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+      } catch (e) {
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+        return null;
+      }
+    }
+    return tx;
+  }
+
+  clearTempCache(): void {
+    for (const txid of this.tempCached) {
+      delete this.txCache[txid];
+    }
+    this.tempCached = [];
+  }
+
+  truncateTempCache(): void {
+    if (this.tempCached.length > tempCacheSize) {
+      const removed = this.tempCached.splice(0, this.tempCached.length - tempCacheSize);
+      for (const txid of removed) {
+        delete this.txCache[txid];
+      }
+    }
+  }
+}
+
+export default new ForensicsService();

backend/src/tasks/lightning/network-sync.service.ts

@@ -0,0 +1,310 @@
+import DB from '../../database';
+import logger from '../../logger';
+import channelsApi from '../../api/explorer/channels.api';
+import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
+import config from '../../config';
+import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
+import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+import { $lookupNodeLocation } from './sync-tasks/node-locations';
+import lightningApi from '../../api/lightning/lightning-api-factory';
+import nodesApi from '../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
+import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
+import { Common } from '../../api/common';
+import blocks from '../../api/blocks';
+import NodeRecordsRepository from '../../repositories/NodeRecordsRepository';
+import forensicsService from './forensics.service';
+
+class NetworkSyncService {
+  loggerTimer = 0;
+  closedChannelsScanBlock = 0;
+
+  constructor() {}
+
+  public async $startService(): Promise<void> {
+    logger.info('Starting lightning network sync service');
+
+    this.loggerTimer = new Date().getTime() / 1000;
+
+    await this.$runTasks();
+  }
+
+  private async $runTasks(): Promise<void> {
+    const taskStartTime = Date.now();
+    try {
+      logger.info(`Updating nodes and channels`);
+
+      const networkGraph = await lightningApi.$getNetworkGraph();
+      if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
+        logger.info(`LN Network graph is empty, retrying in 10 seconds`);
+        setTimeout(() => { this.$runTasks(); }, 10000);
+        return;
+      }
+
+      await this.$updateNodesList(networkGraph.nodes);
+      await this.$updateChannelsList(networkGraph.edges);
+      await this.$deactivateChannelsWithoutActiveNodes();
+      await this.$lookUpCreationDateFromChain();
+      await this.$updateNodeFirstSeen();
+      await this.$scanForClosedChannels();
+      
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        // run forensics on new channels only
+        await forensicsService.$runClosedChannelsForensics(true);
+      }
+
+    } catch (e) {
+      logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
+    }
+
+    setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
+  }
+
+  /**
+   * Update the `nodes` table to reflect the current network graph state
+   */
+  private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
+    let progress = 0;
+
+    let deletedSockets = 0;
+    let deletedRecords = 0;
+    const graphNodesPubkeys: string[] = [];
+    for (const node of nodes) {
+      const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
+      node.last_update = Math.max(node.last_update, latestUpdated);
+
+      await nodesApi.$saveNode(node);
+      graphNodesPubkeys.push(node.pub_key);
+      ++progress;
+
+      const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+      if (elapsedSeconds > 10) {
+        logger.info(`Updating node ${progress}/${nodes.length}`);
+        this.loggerTimer = new Date().getTime() / 1000;
+      }
+
+      const addresses: string[] = [];
+      for (const socket of node.addresses) {
+        await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket));
+        addresses.push(socket.addr);
+      }
+      deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+
+      const oldRecordTypes = await NodeRecordsRepository.$getRecordTypes(node.pub_key);
+      const customRecordTypes: number[] = [];
+      for (const [type, payload] of Object.entries(node.custom_records || {})) {
+        const numericalType = parseInt(type);
+        await NodeRecordsRepository.$saveRecord({
+          publicKey: node.pub_key,
+          type: numericalType,
+          payload,
+        });
+        customRecordTypes.push(numericalType);
+      }
+      if (oldRecordTypes.reduce((changed, type) => changed || customRecordTypes.indexOf(type) === -1, false)) {
+        deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes);
+      }
+    }
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);
+
+    // If a channel if not present in the graph, mark it as inactive
+    await nodesApi.$setNodesInactive(graphNodesPubkeys);
+
+    if (config.MAXMIND.ENABLED) {
+      $lookupNodeLocation();
+    }
+  }
+
+  /**
+   * Update the `channels` table to reflect the current network graph state
+   */
+  private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
+    try {
+      const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`);
+      const closedChannels = {};
+      for (const closedChannel of closedChannelsRaw) {
+        closedChannels[closedChannel.id] = true;
+      }
+
+      let progress = 0;
+
+      const graphChannelsIds: string[] = [];
+      for (const channel of channels) {
+        if (!closedChannels[channel.channel_id]) {
+          await channelsApi.$saveChannel(channel);
+        }
+        graphChannelsIds.push(channel.channel_id);
+        ++progress;
+
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      logger.info(`${progress} channels updated`);
+
+      // If a channel if not present in the graph, mark it as inactive
+      await channelsApi.$setChannelsInactive(graphChannelsIds);
+    } catch (e) {
+      logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
+    }
+  }
+
+  // This method look up the creation date of the earliest channel of the node
+  // and update the node to that date in order to get the earliest first seen date
+  private async $updateNodeFirstSeen(): Promise<void> {
+    let progress = 0;
+    let updated = 0;
+
+    try {
+      const [nodes]: any[] = await DB.query(`
+        SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node1_public_key = nodes.public_key
+        ) AS created1,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node2_public_key = nodes.public_key
+        ) AS created2
+        FROM nodes
+      `);
+
+      for (const node of nodes) {
+        const lowest = Math.min(
+          node.created1 ?? Number.MAX_SAFE_INTEGER,
+          node.created2 ?? Number.MAX_SAFE_INTEGER,
+          node.first_seen ?? Number.MAX_SAFE_INTEGER
+        );
+        if (lowest < node.first_seen) {
+          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
+          const params = [lowest, node.public_key];
+          await DB.query(query, params);
+        }
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+          ++updated;
+        }
+      }
+      logger.info(`Updated ${updated} node first seen dates`);
+    } catch (e) {
+      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $lookUpCreationDateFromChain(): Promise<void> {
+    let progress = 0;
+
+    logger.info(`Running channel creation date lookup`);
+    try {
+      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
+      for (const channel of channels) {
+        const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
+        await DB.query(`
+          UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
+          [transaction.timestamp, channel.id]
+        );
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel creation date ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Updated ${channels.length} channels' creation date`);
+    } catch (e) {
+      logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * If a channel does not have any active node linked to it, then also
+   * mark that channel as inactive
+   */
+  private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
+    logger.info(`Find channels which nodes are offline`);
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE channels.status = 1
+        AND (
+          (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node1_public_key
+            AND nodes.status = 1
+          ) = 0
+        OR (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node2_public_key
+            AND nodes.status = 1
+          ) = 0)
+        `);
+
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      }
+    } catch (e) {
+      logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $scanForClosedChannels(): Promise<void> {
+    if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
+      logger.debug(`We've already scan closed channels for this block, skipping.`);
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      let log = `Starting closed channels scan`;
+      if (this.closedChannelsScanBlock > 0) {
+        log += `. Last scan was at block ${this.closedChannelsScanBlock}`;
+      } else {
+        log += ` for the first time`;
+      }
+      logger.info(log);
+
+      const channels = await channelsApi.$getChannelsByStatus([0, 1]);
+      for (const channel of channels) {
+        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
+        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
+          logger.debug('Marking channel: ' + channel.id + ' as closed.');
+          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
+            [spendingTx.status.block_time, channel.id]);
+          if (spendingTx.txid && !channel.closing_transaction_id) {
+            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+          }
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
+      logger.info(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`);
+    } catch (e) {
+      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new NetworkSyncService();

backend/src/tasks/lightning/stats-updater.service.ts

@@ -0,0 +1,34 @@
+import logger from '../../logger';
+import lightningApi from '../../api/lightning/lightning-api-factory';
+import LightningStatsImporter from './sync-tasks/stats-importer';
+import config from '../../config';
+import { Common } from '../../api/common';
+
+class LightningStatsUpdater {
+  public async $startService(): Promise<void> {
+    logger.info('Starting Lightning Stats service');
+
+    await this.$runTasks();
+    LightningStatsImporter.$run();
+  }
+
+  private async $runTasks(): Promise<void> {
+    await this.$logStatsDaily();
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
+  }
+
+  /**
+   * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
+   */
+  private async $logStatsDaily(): Promise<void> {
+    const date = new Date();
+    Common.setDateMidnight(date);
+    const networkGraph = await lightningApi.$getNetworkGraph();
+    await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
+    
+    logger.info(`Updated latest network stats`);
+  }
+}
+
+export default new LightningStatsUpdater();

backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts

@@ -0,0 +1,116 @@
+import { existsSync, promises } from 'fs';
+import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
+import { Common } from '../../../api/common';
+import config from '../../../config';
+import logger from '../../../logger';
+
+const fsPromises = promises;
+
+const BLOCKS_CACHE_MAX_SIZE = 100;  
+const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json';
+
+class FundingTxFetcher {
+  private running = false;
+  private blocksCache = {};
+  private channelNewlyProcessed = 0;
+  public fundingTxCache = {};
+
+  async $init(): Promise<void> {
+    // Load funding tx disk cache
+    if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) {
+      try {
+        this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
+      } catch (e) {
+        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`);
+        this.fundingTxCache = {};
+      }
+      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`);
+    }
+  }
+
+  async $fetchChannelsFundingTxs(channelIds: string[]): Promise<void> {
+    if (this.running) {
+      return;
+    }
+    this.running = true;
+    
+    const globalTimer = new Date().getTime() / 1000;
+    let cacheTimer = new Date().getTime() / 1000;
+    let loggerTimer = new Date().getTime() / 1000;
+    let channelProcessed = 0;
+    this.channelNewlyProcessed = 0;
+    for (const channelId of channelIds) {
+      await this.$fetchChannelOpenTx(channelId);
+      ++channelProcessed;
+
+      let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+      if (elapsedSeconds > 10) {
+        elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
+        logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
+          `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
+          `elapsed: ${elapsedSeconds} seconds`
+        );
+        loggerTimer = new Date().getTime() / 1000;
+      }
+
+      elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
+      if (elapsedSeconds > 60) {
+        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+        fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+        cacheTimer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (this.channelNewlyProcessed > 0) {
+      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`);
+      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+      fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+    }
+
+    this.running = false;
+  }
+  
+  public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
+    channelId = Common.channelIntegerIdToShortId(channelId);
+
+    if (this.fundingTxCache[channelId]) {
+      return this.fundingTxCache[channelId];
+    }
+
+    const parts = channelId.split('x');
+    const blockHeight = parts[0];
+    const txIdx = parts[1];
+    const outputIdx = parts[2];
+
+    let block = this.blocksCache[blockHeight];
+    // Fetch it from core
+    if (!block) {
+      const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10));
+      block = await bitcoinClient.getBlock(blockHash, 1);
+    }
+    this.blocksCache[block.height] = block;
+
+    const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse();
+    if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) {
+      for (let i = 0; i < 10; ++i) {
+        delete this.blocksCache[blocksCacheHashes[i]];
+      }
+    }
+
+    const txid = block.tx[txIdx];
+    const rawTx = await bitcoinClient.getRawTransaction(txid);
+    const tx = await bitcoinClient.decodeRawTransaction(rawTx);
+
+    this.fundingTxCache[channelId] = {
+      timestamp: block.time,
+      txid: txid,
+      value: tx.vout[outputIdx].value,
+    };
+
+    ++this.channelNewlyProcessed;
+
+    return this.fundingTxCache[channelId];
+  }
+}
+
+export default new FundingTxFetcher;

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -0,0 +1,171 @@
+import * as net from 'net';
+import maxmind, { CityResponse, AsnResponse, IspResponse } from 'maxmind';
+import nodesApi from '../../../api/explorer/nodes.api';
+import config from '../../../config';
+import DB from '../../../database';
+import logger from '../../../logger';
+import { ResultSetHeader } from 'mysql2';
+import * as IPCheck from '../../../utils/ipcheck.js';
+import { Reader } from 'mmdb-lib';
+
+export async function $lookupNodeLocation(): Promise<void> {
+  let loggerTimer = new Date().getTime() / 1000;
+  let progress = 0;
+  let nodesUpdated = 0;
+  let geoNamesInserted = 0;
+
+  logger.info(`Running node location updater using Maxmind`);
+  try {
+    const nodes = await nodesApi.$getAllNodes();
+    const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
+    const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
+    let lookupIsp: Reader<IspResponse> | null = null;
+    try {
+      lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
+    } catch (e) { }
+
+    for (const node of nodes) {
+      const sockets: string[] = node.sockets.split(',');
+      for (const socket of sockets) {
+        const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
+        const hasClearnet = [4, 6].includes(net.isIP(ip));
+
+        if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
+          const city = lookupCity.get(ip);
+          const asn = lookupAsn.get(ip);
+          let isp: IspResponse | null = null;
+          if (lookupIsp) {
+            isp = lookupIsp.get(ip);
+          }
+
+          let asOverwrite: any | undefined;
+          if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
+            asOverwrite = {
+              asn: 394745,
+              name: 'Lunanode',
+            };
+          }
+          else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) {
+            asOverwrite = {
+              asn: 30058,
+              name: 'FDCservers.net',
+            };
+          }
+          else if (asn && asn.autonomous_system_number === 174) {
+            asOverwrite = {
+              asn: 174,
+              name: 'Cogent Communications',
+            };
+          }
+
+          if (city && (asn || isp)) {
+            const query = `
+              UPDATE nodes SET 
+                as_number = ?, 
+                city_id = ?, 
+                country_id = ?, 
+                subdivision_id = ?, 
+                longitude = ?, 
+                latitude = ?, 
+                accuracy_radius = ?
+              WHERE public_key = ?
+            `;
+
+            const params = [
+              asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+              city.city?.geoname_id,
+              city.country?.geoname_id,
+              city.subdivisions ? city.subdivisions[0].geoname_id : null,
+              city.location?.longitude,
+              city.location?.latitude,
+              city.location?.accuracy_radius,
+              node.public_key
+            ];
+            let result = await DB.query<ResultSetHeader>(query, params);
+            if (result[0].changedRows ?? 0 > 0) {
+              ++nodesUpdated;
+            }
+
+            // Store Continent
+            if (city.continent?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
+                [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
+
+            // Store Country
+            if (city.country?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
+                [city.country?.geoname_id, JSON.stringify(city.country?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
+
+            // Store Country ISO code
+            if (city.country?.iso_code) {
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
+                [city.country?.geoname_id, city.country?.iso_code]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
+
+            // Store Division
+            if (city.subdivisions && city.subdivisions[0]) {
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
+                [city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
+
+            // Store City
+            if (city.city?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
+                [city.city?.geoname_id, JSON.stringify(city.city?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
+
+            // Store AS name
+            if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
+                [
+                  asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+                  JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
+                ]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
+          }
+
+          ++progress;
+          const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+          if (elapsedSeconds > 10) {
+            logger.info(`Updating node location data ${progress}/${nodes.length}`);
+            loggerTimer = new Date().getTime() / 1000;
+          }
+        }
+      }
+    }
+
+    if (nodesUpdated > 0) {
+      logger.info(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+    } else {
+      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+    }
+  } catch (e) {
+    logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
+  }
+}

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -0,0 +1,542 @@
+import DB from '../../../database';
+import { promises } from 'fs';
+import logger from '../../../logger';
+import fundingTxFetcher from './funding-tx-fetcher';
+import config from '../../../config';
+import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
+import { isIP } from 'net';
+import { Common } from '../../../api/common';
+import channelsApi from '../../../api/explorer/channels.api';
+import nodesApi from '../../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+
+const fsPromises = promises;
+
+class LightningStatsImporter {
+  topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
+
+  async $run(): Promise<void> {
+    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+    logger.info('Caching funding txs for currently existing channels');
+    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+
+    if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
+      return;
+    }
+
+    await this.$importHistoricalLightningStats();
+    await this.$cleanupIncorrectSnapshot();
+  }
+
+  /**
+   * Generate LN network stats for one day
+   */
+  public async computeNetworkStats(timestamp: number,
+    networkGraph: ILightningApi.NetworkGraph, isHistorical: boolean = false): Promise<unknown> {
+    // Node counts and network shares
+    let clearnetNodes = 0;
+    let torNodes = 0;
+    let clearnetTorNodes = 0;
+    let unannouncedNodes = 0;
+
+    const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`);
+    const nodesInDb = {};
+    for (const node of nodesInDbRaw) {
+      nodesInDb[node.public_key] = node;
+    }
+
+    for (const node of networkGraph.nodes) {
+      // If we don't know about this node, insert it in db
+      if (isHistorical === true && !nodesInDb[node.pub_key]) {
+        await nodesApi.$saveNode({
+          last_update: node.last_update,
+          pub_key: node.pub_key,
+          alias: node.alias,
+          addresses: node.addresses,
+          color: node.color,
+          features: node.features,
+        });
+        nodesInDb[node.pub_key] = node;
+      } else {
+        await nodesApi.$updateNodeSockets(node.pub_key, node.addresses);
+      }
+
+      let hasOnion = false;
+      let hasClearnet = false;
+      let isUnnanounced = true;
+
+      for (const socket of (node.addresses ?? [])) {
+        if (!socket.network?.length && !socket.addr?.length) {
+          continue;
+        }
+        hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1 || socket.addr.indexOf('torv2') !== -1 || socket.addr.indexOf('torv3') !== -1;
+        hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])) || socket.addr.indexOf('ipv4') !== -1 || socket.addr.indexOf('ipv6') !== -1;;
+      }
+      if (hasOnion && hasClearnet) {
+        clearnetTorNodes++;
+        isUnnanounced = false;
+      } else if (hasOnion) {
+        torNodes++;
+        isUnnanounced = false;
+      } else if (hasClearnet) {
+        clearnetNodes++;
+        isUnnanounced = false;
+      }
+      if (isUnnanounced) {
+        unannouncedNodes++;
+      }
+    }
+
+    // Channels and node historical stats
+    const nodeStats = {};
+    let capacity = 0;
+    let avgFeeRate = 0;
+    let avgBaseFee = 0;
+    const capacities: number[] = [];
+    const feeRates: number[] = [];
+    const baseFees: number[] = [];
+    const alreadyCountedChannels = {};
+    
+    const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
+    const channelsInDb = {};
+    for (const channel of channelsInDbRaw) {
+      channelsInDb[channel.short_id] = channel;
+    }
+
+    for (const channel of networkGraph.edges) {
+      const short_id = Common.channelIntegerIdToShortId(channel.channel_id);
+
+      const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
+      if (!tx) {
+        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`);
+        continue;
+      }
+
+      // If we don't know about this channel, insert it in db
+      if (isHistorical === true && !channelsInDb[short_id]) {
+        await channelsApi.$saveChannel({
+          channel_id: short_id,
+          chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
+          last_update: channel.last_update,
+          node1_pub: channel.node1_pub,
+          node2_pub: channel.node2_pub,
+          capacity: (tx.value * 100000000).toString(),
+          node1_policy: null,
+          node2_policy: null,
+        }, 0);
+        channelsInDb[channel.channel_id] = channel;
+      }
+
+      if (!nodeStats[channel.node1_pub]) {
+        nodeStats[channel.node1_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      if (!nodeStats[channel.node2_pub]) {
+        nodeStats[channel.node2_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      
+      if (!alreadyCountedChannels[short_id]) {
+        capacity += Math.round(tx.value * 100000000);
+        capacities.push(Math.round(tx.value * 100000000));
+        alreadyCountedChannels[short_id] = true;
+
+        nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node1_pub].channels++;
+        nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node2_pub].channels++;
+      }
+
+      if (isHistorical === false) { // Coming from the node
+        for (const policy of [channel.node1_policy, channel.node2_policy]) {
+          if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) {
+            avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
+            feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
+          }  
+          if (policy && parseInt(policy.fee_base_msat, 10) < 5000) {
+            avgBaseFee += parseInt(policy.fee_base_msat, 10);
+            baseFees.push(parseInt(policy.fee_base_msat, 10));
+          }
+        }
+      } else {
+        // @ts-ignore
+        if (channel.node1_policy.fee_rate_milli_msat < 5000) {
+          // @ts-ignore
+          avgFeeRate += parseInt(channel.node1_policy.fee_rate_milli_msat, 10);
+          // @ts-ignore
+          feeRates.push(parseInt(channel.node1_policy.fee_rate_milli_msat), 10);
+        }
+        // @ts-ignore
+        if (channel.node1_policy.fee_base_msat < 5000) {
+          // @ts-ignore
+          avgBaseFee += parseInt(channel.node1_policy.fee_base_msat, 10);
+          // @ts-ignore
+          baseFees.push(parseInt(channel.node1_policy.fee_base_msat), 10);
+        }
+      }
+    }
+
+    let medCapacity = 0;
+    let medFeeRate = 0;
+    let medBaseFee = 0;
+    let avgCapacity = 0;
+
+    avgFeeRate /= Math.max(networkGraph.edges.length, 1);
+    avgBaseFee /= Math.max(networkGraph.edges.length, 1);
+
+    if (capacities.length > 0) {
+      medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
+      avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
+    }
+    if (feeRates.length > 0) {
+      medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
+    }
+    if (baseFees.length > 0) {
+      medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
+    }
+
+    let query = `INSERT INTO lightning_stats(
+        added,
+        channel_count,
+        node_count,
+        total_capacity,
+        tor_nodes,
+        clearnet_nodes,
+        unannounced_nodes,
+        clearnet_tor_nodes,
+        avg_capacity,
+        avg_fee_rate,
+        avg_base_fee_mtokens,
+        med_capacity,
+        med_fee_rate,
+        med_base_fee_mtokens
+      )
+      VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+      added = FROM_UNIXTIME(?),
+      channel_count = ?,
+      node_count = ?,
+      total_capacity = ?,
+      tor_nodes = ?,
+      clearnet_nodes = ?,
+      unannounced_nodes = ?,
+      clearnet_tor_nodes = ?,
+      avg_capacity = ?,
+      avg_fee_rate = ?,
+      avg_base_fee_mtokens = ?,
+      med_capacity = ?,
+      med_fee_rate = ?,
+      med_base_fee_mtokens = ?
+    `;
+
+    await DB.query(query, [
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+    ]);
+
+    for (const public_key of Object.keys(nodeStats)) {
+      query = `INSERT INTO node_stats(
+          public_key,
+          added,
+          capacity,
+          channels
+        )
+        VALUES (?, FROM_UNIXTIME(?), ?, ?)
+        ON DUPLICATE KEY UPDATE
+        added = FROM_UNIXTIME(?),
+        capacity = ?,
+        channels = ?
+      `;
+
+      await DB.query(query, [
+        public_key,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+      ]);
+
+      if (!isHistorical) {
+        await DB.query(
+          `UPDATE nodes SET capacity = ?, channels = ? WHERE public_key = ?`,
+          [
+            nodeStats[public_key].capacity,
+            nodeStats[public_key].channels,
+            public_key,
+          ]
+        );
+      }
+    }
+
+    return {
+      added: timestamp,
+      node_count: networkGraph.nodes.length
+    };
+  }
+
+  /**
+   * Import topology files LN historical data into the database
+   */
+  async $importHistoricalLightningStats(): Promise<void> {
+    logger.debug('Run the historical importer');
+    try {
+      let fileList: string[] = [];
+      try {
+        fileList = await fsPromises.readdir(this.topologiesFolder);
+      } catch (e) {
+        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`);
+        throw e;
+      }
+      // Insert history from the most recent to the oldest
+      // This also put the .json cached files first
+      fileList.sort().reverse();
+
+      const [rows]: any[] = await DB.query(`
+        SELECT UNIX_TIMESTAMP(added) AS added
+        FROM lightning_stats
+        ORDER BY added DESC
+      `);
+      const existingStatsTimestamps = {};
+      for (const row of rows) {
+        existingStatsTimestamps[row.added] = row;
+      }
+
+      // For logging purpose
+      let processed = 10;
+      let totalProcessed = 0;
+      let logStarted = false;
+
+      for (const filename of fileList) {
+        processed++;
+
+        const timestamp = parseInt(filename.split('_')[1], 10);
+
+        // Stats exist already, don't calculate/insert them
+        if (existingStatsTimestamps[timestamp] !== undefined) {
+          totalProcessed++;
+          continue;
+        }
+
+        if (filename.indexOf('topology_') === -1) {
+          totalProcessed++;
+          continue;
+        }
+
+        logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
+        let fileContent = '';
+        try {
+          fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
+        } catch (e: any) {
+          if (e.errno == -1) { // EISDIR - Ignore directorie
+            totalProcessed++;
+            continue;
+          }
+          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`);
+          totalProcessed++;
+          continue;
+        }
+
+        let graph;
+        try {
+          graph = JSON.parse(fileContent);
+          graph = await this.cleanupTopology(graph);
+        } catch (e) {
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
+          totalProcessed++;
+          continue;
+        }
+    
+        if (this.isIncorrectSnapshot(timestamp, graph)) {
+          logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
+          ++totalProcessed;
+          continue;
+        }
+
+        if (!logStarted) {
+          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
+          logStarted = true;
+        }
+        
+        const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
+        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
+
+        totalProcessed++;
+
+        if (processed > 10) {
+          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+          processed = 0;
+        } else {
+          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+        }
+        await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
+        const stat = await this.computeNetworkStats(timestamp, graph, true);
+
+        existingStatsTimestamps[timestamp] = stat;
+      }
+
+      if (totalProcessed > 0) {
+        logger.info(`Lightning network stats historical import completed`);
+      }
+    } catch (e) {
+      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  cleanupTopology(graph): ILightningApi.NetworkGraph {
+    const newGraph = {
+      nodes: <ILightningApi.Node[]>[],
+      edges: <ILightningApi.Channel[]>[],
+    };
+
+    for (const node of graph.nodes) {
+      const addressesParts = (node.addresses ?? '').split(',');
+      const addresses: any[] = [];
+      for (const address of addressesParts) {
+        const formatted = Common.findSocketNetwork(address);
+        addresses.push({
+          network: formatted.network,
+          addr: formatted.url
+        });
+      }
+
+      let rgb = node.rgb_color ?? '#000000';
+      if (rgb.indexOf('#') === -1) {
+        rgb = `#${rgb}`;
+      }
+      newGraph.nodes.push({
+        last_update: node.timestamp ?? 0,
+        pub_key: node.id ?? null,
+        alias: node.alias ?? node.id.slice(0, 20),
+        addresses: addresses,
+        color: rgb,
+        features: {},
+      });
+    }
+
+    for (const adjacency of graph.adjacency) {
+      if (adjacency.length === 0) {
+        continue;
+      } else {
+        for (const edge of adjacency) {
+          newGraph.edges.push({
+            channel_id: edge.scid,
+            chan_point: '',
+            last_update: edge.timestamp,
+            node1_pub: edge.source ?? null,
+            node2_pub: edge.destination ?? null,
+            capacity: '0', // Will be fetch later
+            node1_policy: {
+              time_lock_delta: edge.cltv_expiry_delta,
+              min_htlc: edge.htlc_minimim_msat,
+              fee_base_msat: edge.fee_base_msat,
+              fee_rate_milli_msat: edge.fee_proportional_millionths,
+              max_htlc_msat: edge.htlc_maximum_msat,
+              last_update: edge.timestamp,
+              disabled: false,          
+            },
+            node2_policy: null,
+          });
+        }
+      }
+    }
+
+    return newGraph;
+  }
+
+  private isIncorrectSnapshot(timestamp, graph): boolean {
+    if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
+        return true;
+    }
+    if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
+      return true;
+    }
+    if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
+      return true;
+    }
+    if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
+      return true;
+    }
+    if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
+      return true;
+    }
+    if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private async $cleanupIncorrectSnapshot(): Promise<void> {
+    // We do not run this one automatically because those stats are not supposed to be inserted in the first
+    // place, but I write them here to remind us we manually run those queries
+
+    // DELETE FROM lightning_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
+    // )
+
+    // DELETE FROM node_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 
+    // )
+  }
+}
+
+export default new LightningStatsImporter;

backend/src/tasks/pools-updater.ts

@@ -12,12 +12,11 @@ import * as https from 'https';
  */
 class PoolsUpdater {
   lastRun: number = 0;
-  currentSha: any = undefined;
+  currentSha: string | undefined = undefined;
+  poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
+  treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
-  constructor() {
-  }
-
-  public async updatePoolsJson() {
+  public async updatePoolsJson(): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -32,11 +31,10 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
-    logger.info('Updating latest mining pools from Github');
     if (config.SOCKS5PROXY.ENABLED) {
-      logger.info('List of public pools will be queried over the Tor network');
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`);
     } else {
-      logger.info('List of public pools will be queried over clearnet');
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`);
     }
 
     try {
@@ -46,7 +44,7 @@ class PoolsUpdater {
       }
 
       if (config.DATABASE.ENABLED === true) {
-        this.currentSha = await this.getShaFromDb();      
+        this.currentSha = await this.getShaFromDb();
       }
 
       logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
@@ -54,8 +52,12 @@ class PoolsUpdater {
         return;
       }
 
-      logger.warn('Pools.json is outdated, fetch latest from github');
-      const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json');
+      if (this.currentSha === undefined) {
+        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`);
+      } else {
+        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`);
+      }
+      const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
         return;
       }
@@ -65,21 +67,21 @@ class PoolsUpdater {
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: '  + (e instanceof Error ? e.message : e));
+      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
 
   /**
    * Fetch our latest pools.json sha from the db
    */
-  private async updateDBSha(githubSha: string) {
+  private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
     if (config.DATABASE.ENABLED === true) {
       try {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: '  + (e instanceof Error ? e.message : e));
+        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
       }
     }
   }
@@ -92,7 +94,7 @@ class PoolsUpdater {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : undefined);
     } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: '  + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
       return undefined;
     }
   }
@@ -101,7 +103,7 @@ class PoolsUpdater {
    * Fetch our latest pools.json sha from github
    */
   private async fetchPoolsSha(): Promise<string | undefined> {
-    const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master');
+    const response = await this.query(this.treeUrl);
 
     if (response !== undefined) {
       for (const file of response['tree']) {
@@ -111,7 +113,7 @@ class PoolsUpdater {
       }
     }
 
-    logger.err('Cannot to find latest pools.json sha from github api response');
+    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`);
     return undefined;
   }
 
@@ -125,7 +127,7 @@ class PoolsUpdater {
       };
       timeout: number;
       httpsAgent?: https.Agent;
-    }
+    };
     const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
     const axiosOptions: axiosOptions = {
       headers: {
@@ -135,7 +137,7 @@ class PoolsUpdater {
     };
     let retry = 0;
 
-    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+    while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
       try {
         if (config.SOCKS5PROXY.ENABLED) {
           const socksOptions: any = {
@@ -156,14 +158,14 @@ class PoolsUpdater {
 
           axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
         }
-        
+
         const data: AxiosResponse = await axios.get(path, axiosOptions);
         if (data.statusText === 'error' || !data.data) {
-          throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
+          throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
         }
         return data.data;
       } catch (e) {
-        logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
+        logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
         retry++;
       }
       await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class BitfinexApi implements PriceFeed {
+  public name: string = 'Bitfinex';
+  public currencies: string[] = ['USD', 'EUR', 'GPB', 'JPY'];
+
+  public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
+  public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['last_price'], 10) : -1;
+  }
+
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1h' : '1D').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price[0] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price[2];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default BitfinexApi;

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -0,0 +1,24 @@
+import { query } from '../../utils/axios-query';
+import { PriceFeed, PriceHistory } from '../price-updater';
+
+class BitflyerApi implements PriceFeed {
+  public name: string = 'Bitflyer';
+  public currencies: string[] = ['USD', 'EUR', 'JPY'];
+
+  public url: string = 'https://api.bitflyer.com/v1/ticker?product_code=BTC_';
+  public urlHist: string = '';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['ltp'], 10) : -1;
+  }
+
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
+    return [];
+  }
+}
+
+export default BitflyerApi;

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -0,0 +1,42 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class CoinbaseApi implements PriceFeed {
+  public name: string = 'Coinbase';
+  public currencies: string[] = ['USD', 'EUR', 'GBP'];
+
+  public url: string = 'https://api.coinbase.com/v2/prices/spot?currency=';
+  public urlHist: string = 'https://api.exchange.coinbase.com/products/BTC-{CURRENCY}/candles?granularity={GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['data']['amount'], 10) : -1;
+  }
+
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default CoinbaseApi;

backend/src/tasks/price-feeds/gemini-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class GeminiApi implements PriceFeed {
+  public name: string = 'Gemini';
+  public currencies: string[] = ['USD', 'EUR', 'GBP', 'SGD'];
+
+  public url: string = 'https://api.gemini.com/v1/pubticker/BTC';
+  public urlHist: string = 'https://api.gemini.com/v2/candles/BTC{CURRENCY}/{GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['last'], 10) : -1;
+  }
+
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1hr' : '1day').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price[0] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default GeminiApi;

backend/src/tasks/price-feeds/kraken-api.ts

@@ -0,0 +1,99 @@
+import logger from '../../logger';
+import PricesRepository from '../../repositories/PricesRepository';
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class KrakenApi implements PriceFeed {
+  public name: string = 'Kraken';
+  public currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+
+  public url: string = 'https://api.kraken.com/0/public/Ticker?pair=XBT';
+  public urlHist: string = 'https://api.kraken.com/0/public/OHLC?interval={GRANULARITY}&pair=XBT';
+
+  constructor() {
+  }
+
+  private getTicker(currency) {
+    let ticker = `XXBTZ${currency}`;
+    if (['CHF', 'AUD'].includes(currency)) {
+      ticker = `XBT${currency}`;
+    }
+    return ticker;
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
+  }
+
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '60') + currency);
+      const pricesRaw = response ? response['result'][this.getTicker(currency)] : [];
+
+      for (const price of pricesRaw) {
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+
+  /**
+   * Fetch weekly price and save it into the database
+   */
+  public async $insertHistoricalPrice(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    // EUR weekly price history goes back to timestamp 1378339200 (September 5, 2013)
+    // USD weekly price history goes back to timestamp 1380758400 (October 3, 2013)
+    // GBP weekly price history goes back to timestamp 1415232000 (November 6, 2014)
+    // JPY weekly price history goes back to timestamp 1415232000 (November 6, 2014)
+    // CAD weekly price history goes back to timestamp 1436400000 (July 9, 2015)
+    // CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019)
+    // AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020)
+
+    let priceHistory: any = {}; // map: timestamp -> Prices
+
+    for (const currency of this.currencies) {
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency);
+      const priceHistoryRaw = response ? response['result'][this.getTicker(currency)] : [];
+
+      for (const price of priceHistoryRaw) {
+        if (existingPriceTimes.includes(parseInt(price[0]))) {
+          continue;
+        }
+
+        // prices[0] = kraken price timestamp
+        // prices[4] = closing price
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    for (const time in priceHistory) {
+      if (priceHistory[time].USD === -1) {
+        delete priceHistory[time];
+        continue;
+      }
+      await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]);
+    }
+
+    if (Object.keys(priceHistory).length > 0) {
+      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
+    }
+  }
+}
+
+export default KrakenApi;

backend/src/tasks/price-feeds/mtgox-weekly.json

@@ -0,0 +1,762 @@
+[
+  {
+    "ct": 1279497600,
+    "c": "0.08584"
+  },
+  {
+    "ct": 1280102400,
+    "c": "0.0505"
+  },
+  {
+    "ct": 1280707200,
+    "c": "0.0611"
+  },
+  {
+    "ct": 1281312000,
+    "c": "0.0609"
+  },
+  {
+    "ct": 1281916800,
+    "c": "0.06529"
+  },
+  {
+    "ct": 1282521600,
+    "c": "0.066"
+  },
+  {
+    "ct": 1283126400,
+    "c": "0.064"
+  },
+  {
+    "ct": 1283731200,
+    "c": "0.06165"
+  },
+  {
+    "ct": 1284336000,
+    "c": "0.0615"
+  },
+  {
+    "ct": 1284940800,
+    "c": "0.0627"
+  },
+  {
+    "ct": 1285545600,
+    "c": "0.0622"
+  },
+  {
+    "ct": 1286150400,
+    "c": "0.06111"
+  },
+  {
+    "ct": 1286755200,
+    "c": "0.0965"
+  },
+  {
+    "ct": 1287360000,
+    "c": "0.102"
+  },
+  {
+    "ct": 1287964800,
+    "c": "0.11501"
+  },
+  {
+    "ct": 1288569600,
+    "c": "0.1925"
+  },
+  {
+    "ct": 1289174400,
+    "c": "0.34"
+  },
+  {
+    "ct": 1289779200,
+    "c": "0.27904"
+  },
+  {
+    "ct": 1290384000,
+    "c": "0.27675"
+  },
+  {
+    "ct": 1290988800,
+    "c": "0.27"
+  },
+  {
+    "ct": 1291593600,
+    "c": "0.19"
+  },
+  {
+    "ct": 1292198400,
+    "c": "0.2189"
+  },
+  {
+    "ct": 1292803200,
+    "c": "0.2401"
+  },
+  {
+    "ct": 1293408000,
+    "c": "0.263"
+  },
+  {
+    "ct": 1294012800,
+    "c": "0.29997"
+  },
+  {
+    "ct": 1294617600,
+    "c": "0.323"
+  },
+  {
+    "ct": 1295222400,
+    "c": "0.38679"
+  },
+  {
+    "ct": 1295827200,
+    "c": "0.4424"
+  },
+  {
+    "ct": 1296432000,
+    "c": "0.4799"
+  },
+  {
+    "ct": 1297036800,
+    "c": "0.8968"
+  },
+  {
+    "ct": 1297641600,
+    "c": "1.05"
+  },
+  {
+    "ct": 1298246400,
+    "c": "0.865"
+  },
+  {
+    "ct": 1298851200,
+    "c": "0.89"
+  },
+  {
+    "ct": 1299456000,
+    "c": "0.8999"
+  },
+  {
+    "ct": 1300060800,
+    "c": "0.89249"
+  },
+  {
+    "ct": 1300665600,
+    "c": "0.75218"
+  },
+  {
+    "ct": 1301270400,
+    "c": "0.82754"
+  },
+  {
+    "ct": 1301875200,
+    "c": "0.779"
+  },
+  {
+    "ct": 1302480000,
+    "c": "0.7369"
+  },
+  {
+    "ct": 1303084800,
+    "c": "1.1123"
+  },
+  {
+    "ct": 1303689600,
+    "c": "1.6311"
+  },
+  {
+    "ct": 1304294400,
+    "c": "3.03311"
+  },
+  {
+    "ct": 1304899200,
+    "c": "3.8659"
+  },
+  {
+    "ct": 1305504000,
+    "c": "6.98701"
+  },
+  {
+    "ct": 1306108800,
+    "c": "6.6901"
+  },
+  {
+    "ct": 1306713600,
+    "c": "8.4"
+  },
+  {
+    "ct": 1307318400,
+    "c": "16.7"
+  },
+  {
+    "ct": 1307923200,
+    "c": "18.5464"
+  },
+  {
+    "ct": 1308528000,
+    "c": "17.51"
+  },
+  {
+    "ct": 1309132800,
+    "c": "16.45001"
+  },
+  {
+    "ct": 1309737600,
+    "c": "15.44049"
+  },
+  {
+    "ct": 1310342400,
+    "c": "14.879"
+  },
+  {
+    "ct": 1310947200,
+    "c": "13.16"
+  },
+  {
+    "ct": 1311552000,
+    "c": "13.98001"
+  },
+  {
+    "ct": 1312156800,
+    "c": "13.35"
+  },
+  {
+    "ct": 1312761600,
+    "c": "7.9"
+  },
+  {
+    "ct": 1313366400,
+    "c": "10.7957"
+  },
+  {
+    "ct": 1313971200,
+    "c": "11.31125"
+  },
+  {
+    "ct": 1314576000,
+    "c": "9.07011"
+  },
+  {
+    "ct": 1315180800,
+    "c": "8.17798"
+  },
+  {
+    "ct": 1315785600,
+    "c": "5.86436"
+  },
+  {
+    "ct": 1316390400,
+    "c": "5.2"
+  },
+  {
+    "ct": 1316995200,
+    "c": "5.33"
+  },
+  {
+    "ct": 1317600000,
+    "c": "5.02701"
+  },
+  {
+    "ct": 1318204800,
+    "c": "4.10288"
+  },
+  {
+    "ct": 1318809600,
+    "c": "3.5574"
+  },
+  {
+    "ct": 1319414400,
+    "c": "3.12657"
+  },
+  {
+    "ct": 1320019200,
+    "c": "3.27"
+  },
+  {
+    "ct": 1320624000,
+    "c": "2.95959"
+  },
+  {
+    "ct": 1321228800,
+    "c": "2.99626"
+  },
+  {
+    "ct": 1321833600,
+    "c": "2.2"
+  },
+  {
+    "ct": 1322438400,
+    "c": "2.47991"
+  },
+  {
+    "ct": 1323043200,
+    "c": "2.82809"
+  },
+  {
+    "ct": 1323648000,
+    "c": "3.2511"
+  },
+  {
+    "ct": 1324252800,
+    "c": "3.193"
+  },
+  {
+    "ct": 1324857600,
+    "c": "4.225"
+  },
+  {
+    "ct": 1325462400,
+    "c": "5.26766"
+  },
+  {
+    "ct": 1326067200,
+    "c": "7.11358"
+  },
+  {
+    "ct": 1326672000,
+    "c": "7.00177"
+  },
+  {
+    "ct": 1327276800,
+    "c": "6.3097"
+  },
+  {
+    "ct": 1327881600,
+    "c": "5.38191"
+  },
+  {
+    "ct": 1328486400,
+    "c": "5.68881"
+  },
+  {
+    "ct": 1329091200,
+    "c": "5.51468"
+  },
+  {
+    "ct": 1329696000,
+    "c": "4.38669"
+  },
+  {
+    "ct": 1330300800,
+    "c": "4.922"
+  },
+  {
+    "ct": 1330905600,
+    "c": "4.8201"
+  },
+  {
+    "ct": 1331510400,
+    "c": "4.90901"
+  },
+  {
+    "ct": 1332115200,
+    "c": "5.27943"
+  },
+  {
+    "ct": 1332720000,
+    "c": "4.55001"
+  },
+  {
+    "ct": 1333324800,
+    "c": "4.81922"
+  },
+  {
+    "ct": 1333929600,
+    "c": "4.79253"
+  },
+  {
+    "ct": 1334534400,
+    "c": "4.96892"
+  },
+  {
+    "ct": 1335139200,
+    "c": "5.20352"
+  },
+  {
+    "ct": 1335744000,
+    "c": "4.90441"
+  },
+  {
+    "ct": 1336348800,
+    "c": "5.04991"
+  },
+  {
+    "ct": 1336953600,
+    "c": "4.92996"
+  },
+  {
+    "ct": 1337558400,
+    "c": "5.09002"
+  },
+  {
+    "ct": 1338163200,
+    "c": "5.13896"
+  },
+  {
+    "ct": 1338768000,
+    "c": "5.2051"
+  },
+  {
+    "ct": 1339372800,
+    "c": "5.46829"
+  },
+  {
+    "ct": 1339977600,
+    "c": "6.16382"
+  },
+  {
+    "ct": 1340582400,
+    "c": "6.35002"
+  },
+  {
+    "ct": 1341187200,
+    "c": "6.62898"
+  },
+  {
+    "ct": 1341792000,
+    "c": "6.79898"
+  },
+  {
+    "ct": 1342396800,
+    "c": "7.62101"
+  },
+  {
+    "ct": 1343001600,
+    "c": "8.4096"
+  },
+  {
+    "ct": 1343606400,
+    "c": "8.71027"
+  },
+  {
+    "ct": 1344211200,
+    "c": "10.86998"
+  },
+  {
+    "ct": 1344816000,
+    "c": "11.6239"
+  },
+  {
+    "ct": 1345420800,
+    "c": "7.98"
+  },
+  {
+    "ct": 1346025600,
+    "c": "10.61"
+  },
+  {
+    "ct": 1346630400,
+    "c": "10.2041"
+  },
+  {
+    "ct": 1347235200,
+    "c": "11.02"
+  },
+  {
+    "ct": 1347840000,
+    "c": "11.87"
+  },
+  {
+    "ct": 1348444800,
+    "c": "12.19331"
+  },
+  {
+    "ct": 1349049600,
+    "c": "12.4"
+  },
+  {
+    "ct": 1349654400,
+    "c": "11.8034"
+  },
+  {
+    "ct": 1350259200,
+    "c": "11.7389"
+  },
+  {
+    "ct": 1350864000,
+    "c": "11.63107"
+  },
+  {
+    "ct": 1351468800,
+    "c": "10.69998"
+  },
+  {
+    "ct": 1352073600,
+    "c": "10.80011"
+  },
+  {
+    "ct": 1352678400,
+    "c": "10.84692"
+  },
+  {
+    "ct": 1353283200,
+    "c": "11.65961"
+  },
+  {
+    "ct": 1353888000,
+    "c": "12.4821"
+  },
+  {
+    "ct": 1354492800,
+    "c": "12.50003"
+  },
+  {
+    "ct": 1355097600,
+    "c": "13.388"
+  },
+  {
+    "ct": 1355702400,
+    "c": "13.30002"
+  },
+  {
+    "ct": 1356307200,
+    "c": "13.31202"
+  },
+  {
+    "ct": 1356912000,
+    "c": "13.45001"
+  },
+  {
+    "ct": 1357516800,
+    "c": "13.5199"
+  },
+  {
+    "ct": 1358121600,
+    "c": "14.11601"
+  },
+  {
+    "ct": 1358726400,
+    "c": "15.7"
+  },
+  {
+    "ct": 1359331200,
+    "c": "17.95"
+  },
+  {
+    "ct": 1359936000,
+    "c": "20.59"
+  },
+  {
+    "ct": 1360540800,
+    "c": "23.96975"
+  },
+  {
+    "ct": 1361145600,
+    "c": "26.8146"
+  },
+  {
+    "ct": 1361750400,
+    "c": "29.88999"
+  },
+  {
+    "ct": 1362355200,
+    "c": "34.49999"
+  },
+  {
+    "ct": 1362960000,
+    "c": "46"
+  },
+  {
+    "ct": 1363564800,
+    "c": "47.4"
+  },
+  {
+    "ct": 1364169600,
+    "c": "71.93"
+  },
+  {
+    "ct": 1364774400,
+    "c": "93.03001"
+  },
+  {
+    "ct": 1365379200,
+    "c": "162.30102"
+  },
+  {
+    "ct": 1365984000,
+    "c": "89.99999"
+  },
+  {
+    "ct": 1366588800,
+    "c": "119.2"
+  },
+  {
+    "ct": 1367193600,
+    "c": "134.44444"
+  },
+  {
+    "ct": 1367798400,
+    "c": "115.98"
+  },
+  {
+    "ct": 1368403200,
+    "c": "114.82002"
+  },
+  {
+    "ct": 1369008000,
+    "c": "122.49999"
+  },
+  {
+    "ct": 1369612800,
+    "c": "133.5"
+  },
+  {
+    "ct": 1370217600,
+    "c": "122.5"
+  },
+  {
+    "ct": 1370822400,
+    "c": "100.43743"
+  },
+  {
+    "ct": 1371427200,
+    "c": "99.9"
+  },
+  {
+    "ct": 1372032000,
+    "c": "107.90001"
+  },
+  {
+    "ct": 1372636800,
+    "c": "97.51"
+  },
+  {
+    "ct": 1373241600,
+    "c": "76.5"
+  },
+  {
+    "ct": 1373846400,
+    "c": "94.41986"
+  },
+  {
+    "ct": 1374451200,
+    "c": "91.998"
+  },
+  {
+    "ct": 1375056000,
+    "c": "98.78008"
+  },
+  {
+    "ct": 1375660800,
+    "c": "105.12"
+  },
+  {
+    "ct": 1376265600,
+    "c": "105"
+  },
+  {
+    "ct": 1376870400,
+    "c": "113.38"
+  },
+  {
+    "ct": 1377475200,
+    "c": "122.11102"
+  },
+  {
+    "ct": 1378080000,
+    "c": "146.01003"
+  },
+  {
+    "ct": 1378684800,
+    "c": "126.31501"
+  },
+  {
+    "ct": 1379289600,
+    "c": "138.3002"
+  },
+  {
+    "ct": 1379894400,
+    "c": "134.00001"
+  },
+  {
+    "ct": 1380499200,
+    "c": "143.88402"
+  },
+  {
+    "ct": 1381104000,
+    "c": "137.8"
+  },
+  {
+    "ct": 1381708800,
+    "c": "147.53"
+  },
+  {
+    "ct": 1382313600,
+    "c": "186.1"
+  },
+  {
+    "ct": 1382918400,
+    "c": "207.0001"
+  },
+  {
+    "ct": 1383523200,
+    "c": "224.01001"
+  },
+  {
+    "ct": 1384128000,
+    "c": "336.33101"
+  },
+  {
+    "ct": 1384732800,
+    "c": "528"
+  },
+  {
+    "ct": 1385337600,
+    "c": "795"
+  },
+  {
+    "ct": 1385942400,
+    "c": "1004.42392"
+  },
+  {
+    "ct": 1386547200,
+    "c": "804.5"
+  },
+  {
+    "ct": 1387152000,
+    "c": "919.985"
+  },
+  {
+    "ct": 1387756800,
+    "c": "639.48"
+  },
+  {
+    "ct": 1388361600,
+    "c": "786.98"
+  },
+  {
+    "ct": 1388966400,
+    "c": "1015"
+  },
+  {
+    "ct": 1389571200,
+    "c": "940"
+  },
+  {
+    "ct": 1390176000,
+    "c": "954.995"
+  },
+  {
+    "ct": 1390780800,
+    "c": "1007.98999"
+  },
+  {
+    "ct": 1391385600,
+    "c": "954"
+  },
+  {
+    "ct": 1391990400,
+    "c": "659.49776"
+  },
+  {
+    "ct": 1392595200,
+    "c": "299.702"
+  },
+  {
+    "ct": 1393200000,
+    "c": "310.00001"
+  },
+  {
+    "ct": 1393804800,
+    "c": "135"
+  }
+]

backend/src/tasks/price-updater.ts

@@ -0,0 +1,262 @@
+import * as fs from 'fs';
+import path from "path";
+import config from '../config';
+import logger from '../logger';
+import PricesRepository from '../repositories/PricesRepository';
+import BitfinexApi from './price-feeds/bitfinex-api';
+import BitflyerApi from './price-feeds/bitflyer-api';
+import CoinbaseApi from './price-feeds/coinbase-api';
+import GeminiApi from './price-feeds/gemini-api';
+import KrakenApi from './price-feeds/kraken-api';
+
+export interface PriceFeed {
+  name: string;
+  url: string;
+  urlHist: string;
+  currencies: string[];
+
+  $fetchPrice(currency): Promise<number>;
+  $fetchRecentPrice(currencies: string[], type: string): Promise<PriceHistory>;
+}
+
+export interface PriceHistory {
+  [timestamp: number]: Prices;
+}
+
+export interface Prices {
+  USD: number;
+  EUR: number;
+  GBP: number;
+  CAD: number;
+  CHF: number;
+  AUD: number;
+  JPY: number;
+}
+
+class PriceUpdater {
+  public historyInserted = false;
+  lastRun = 0;
+  lastHistoricalRun = 0;
+  running = false;
+  feeds: PriceFeed[] = [];
+  currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+  latestPrices: Prices;
+
+  constructor() {
+    this.latestPrices = this.getEmptyPricesObj();
+
+    this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
+    this.feeds.push(new KrakenApi());
+    this.feeds.push(new CoinbaseApi());
+    this.feeds.push(new BitfinexApi());
+    this.feeds.push(new GeminiApi());
+  }
+
+  public getEmptyPricesObj(): Prices {
+    return {
+      USD: -1,
+      EUR: -1,
+      GBP: -1,
+      CAD: -1,
+      CHF: -1,
+      AUD: -1,
+      JPY: -1,
+    };
+  }
+
+  public async $run(): Promise<void> {
+    if (this.running === true) {
+      return;
+    }
+    this.running = true;
+
+    if ((Math.round(new Date().getTime() / 1000) - this.lastHistoricalRun) > 3600 * 24) {
+      // Once a day, look for missing prices (could happen due to network connectivity issues)
+      this.historyInserted = false;
+    }
+
+    try {
+      if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
+        await this.$insertHistoricalPrices();
+      } else {
+        await this.$updatePrice();
+      }
+    } catch (e) {
+      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+
+    this.running = false;
+  }
+
+  /**
+   * Fetch last BTC price from exchanges, average them, and save it in the database once every hour
+   */
+  private async $updatePrice(): Promise<void> {
+    if (this.lastRun === 0 && config.DATABASE.ENABLED === true) {
+      this.lastRun = await PricesRepository.$getLatestPriceTime();
+    }
+
+    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) {
+      // Refresh only once every hour
+      return;
+    }
+
+    const previousRun = this.lastRun;
+    this.lastRun = new Date().getTime() / 1000;
+
+    for (const currency of this.currencies) {
+      let prices: number[] = [];
+
+      for (const feed of this.feeds) {
+        // Fetch prices from API which supports `currency`
+        if (feed.currencies.includes(currency)) {
+          try {
+            const price = await feed.$fetchPrice(currency);
+            if (price > 0) {
+              prices.push(price);
+            }
+            logger.debug(`${feed.name} BTC/${currency} price: ${price}`);
+          } catch (e) {
+            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`);
+          }
+        }
+      }
+      if (prices.length === 1) {
+        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`);
+      }
+
+      // Compute average price, non weighted
+      prices = prices.filter(price => price > 0);
+      this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+    }
+
+    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
+
+    if (config.DATABASE.ENABLED === true) {
+      // Save everything in db
+      try {
+        const p = 60 * 60 * 1000; // milliseconds in an hour
+        const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
+        await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
+      } catch (e) {
+        this.lastRun = previousRun + 5 * 60;
+        logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
+      }
+    }
+
+    this.lastRun = new Date().getTime() / 1000;
+  }
+
+  /**
+   * Called once by the database migration to initialize historical prices data (weekly)
+   * We use MtGox weekly price from July 19, 2010 to September 30, 2013
+   * We use Kraken weekly price from October 3, 2013 up to last month
+   * We use Kraken hourly price for the past month
+   */
+  private async $insertHistoricalPrices(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    // Insert MtGox weekly prices
+    const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
+    const prices = this.getEmptyPricesObj();
+    let insertedCount: number = 0;
+    for (const price of pricesJson) {
+      if (existingPriceTimes.includes(price['ct'])) {
+        continue;
+      }
+
+      // From 1380758400 we will use Kraken price as it follows closely MtGox, but was not affected as much
+      // by the MtGox exchange collapse a few months later
+      if (price['ct'] > 1380758400) {
+        break;
+      }
+      prices.USD = price['c'];
+      await PricesRepository.$savePrices(price['ct'], prices);
+      ++insertedCount;
+    }
+    if (insertedCount > 0) {
+      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+    } else {
+      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+    }
+
+    // Insert Kraken weekly prices
+    await new KrakenApi().$insertHistoricalPrice();
+
+    // Insert missing recent hourly prices
+    await this.$insertMissingRecentPrices('day');
+    await this.$insertMissingRecentPrices('hour');
+
+    this.historyInserted = true;
+    this.lastHistoricalRun = new Date().getTime();
+  }
+
+  /**
+   * Find missing hourly prices and insert them in the database
+   * It has a limited backward range and it depends on which API are available
+   */
+  private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database, this may take a while`);
+
+    const historicalPrices: PriceHistory[] = [];
+
+    // Fetch all historical hourly prices
+    for (const feed of this.feeds) {
+      try {
+        historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type));
+      } catch (e) {
+        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
+      }
+    }
+
+    // Group them by timestamp and currency, for example
+    // grouped[123456789]['USD'] = [1, 2, 3, 4];
+    const grouped: Object = {};
+    for (const historicalEntry of historicalPrices) {
+      for (const time in historicalEntry) {
+        if (existingPriceTimes.includes(parseInt(time, 10))) {
+          continue;
+        }
+
+        if (grouped[time] === undefined) {
+          grouped[time] = {
+            USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: []
+          };
+        }
+
+        for (const currency of this.currencies) {
+          const price = historicalEntry[time][currency];
+          if (price > 0) {
+            grouped[time][currency].push(parseInt(price, 10));
+          }
+        }
+      }
+    }
+
+    // Average prices and insert everything into the db
+    let totalInserted = 0;
+    for (const time in grouped) {
+      const prices: Prices = this.getEmptyPricesObj();
+      for (const currency in grouped[time]) {
+        if (grouped[time][currency].length === 0) {
+          continue;
+        }
+        prices[currency] = Math.round((grouped[time][currency].reduce(
+          (partialSum, a) => partialSum + a, 0)
+        ) / grouped[time][currency].length);
+      }
+      await PricesRepository.$savePrices(parseInt(time, 10), prices);
+      ++totalInserted;
+    }
+
+    if (totalInserted > 0) {
+      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
+    } else {
+      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
+    }
+  }
+}
+
+export default new PriceUpdater();

backend/src/utils/axios-query.ts

@@ -0,0 +1,63 @@
+import axios, { AxiosResponse } from 'axios';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import backendInfo from '../api/backend-info';
+import config from '../config';
+import logger from '../logger';
+import * as https from 'https';
+
+export async function query(path): Promise<object | undefined> {
+ type axiosOptions = {
+   headers: {
+     'User-Agent': string
+   };
+   timeout: number;
+   httpsAgent?: https.Agent;
+ };
+ const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+ const axiosOptions: axiosOptions = {
+   headers: {
+     'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+   },
+   timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+ };
+ let retry = 0;
+
+ while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+   try {
+     if (config.SOCKS5PROXY.ENABLED) {
+       const socksOptions: any = {
+         agentOptions: {
+           keepAlive: true,
+         },
+         hostname: config.SOCKS5PROXY.HOST,
+         port: config.SOCKS5PROXY.PORT
+       };
+
+       if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+         socksOptions.username = config.SOCKS5PROXY.USERNAME;
+         socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+       } else {
+         // Retry with different tor circuits https://stackoverflow.com/a/64960234
+         socksOptions.username = `circuit${retry}`;
+       }
+
+       axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+     }
+
+     const data: AxiosResponse = await axios.get(path, axiosOptions);
+     if (data.statusText === 'error' || !data.data) {
+       throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
+     }
+     return data.data;
+   } catch (e) {
+     logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
+     retry++;
+   }
+   if (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+     await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+   }
+ }
+
+ logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
+ return undefined;
+}

backend/src/utils/blocks-utils.ts

@@ -3,14 +3,14 @@ import { BlockExtended } from '../mempool.interfaces';
 export function prepareBlock(block: any): BlockExtended {
   return <BlockExtended>{
     id: block.id ?? block.hash, // hash for indexed block
-    timestamp: block.timestamp ?? block.blockTimestamp, // blockTimestamp for indexed block
+    timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
     height: block.height,
     version: block.version,
-    bits: block.bits,
+    bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
     nonce: block.nonce,
     difficulty: block.difficulty,
-    merkle_root: block.merkle_root,
-    tx_count: block.tx_count,
+    merkle_root: block.merkle_root ?? block.merkleroot,
+    tx_count: block.tx_count ?? block.nTx,
     size: block.size,
     weight: block.weight,
     previousblockhash: block.previousblockhash,
@@ -27,6 +27,7 @@ export function prepareBlock(block: any): BlockExtended {
         name: block.pool_name,
         slug: block.pool_slug,
       } : undefined),
+      usd: block?.extras?.usd ?? block.usd ?? null,
     }
   };
 }

backend/src/utils/ipcheck.js

@@ -0,0 +1,119 @@
+var net = require('net');
+
+var IPCheck = module.exports = function(input) {
+  var self = this;
+
+  if (!(self instanceof IPCheck)) {
+    return new IPCheck(input);
+  }
+
+  self.input = input;
+  self.parse();
+};
+
+IPCheck.prototype.parse = function() {
+  var self = this;
+
+  if (!self.input || typeof self.input !== 'string') return self.valid = false;
+
+  var ip;
+
+  var pos = self.input.lastIndexOf('/');
+  if (pos !== -1) {
+    ip = self.input.substring(0, pos);
+    self.mask = +self.input.substring(pos + 1);
+  } else {
+    ip = self.input;
+    self.mask = null;
+  }
+
+  self.ipv = net.isIP(ip);
+  self.valid = !!self.ipv && !isNaN(self.mask);
+
+  if (!self.valid) return;
+
+  // default mask = 32 for ipv4 and 128 for ipv6
+  if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
+
+  if (self.ipv === 4) {
+    // difference between ipv4 and ipv6 masks
+    self.mask += 96;
+  }
+
+  if (self.mask < 0 || self.mask > 128) {
+    self.valid = false;
+    return;
+  }
+
+  self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
+
+  if(self.ipv === 4){
+    self.parseIPv4(ip);
+  }else{
+    self.parseIPv6(ip);
+  }
+};
+
+IPCheck.prototype.parseIPv4 = function(ip) {
+  var self = this;
+
+  // ipv4 addresses live under ::ffff:0:0
+  self.address[10] = self.address[11] = 0xff;
+
+  var octets = ip.split('.');
+  for (var i = 0; i < 4; i++) {
+    self.address[i + 12] = parseInt(octets[i], 10);
+  }
+};
+
+
+var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
+
+IPCheck.prototype.parseIPv6 = function(ip) {
+  var self = this;
+
+  var transitionalMatch = V6_TRANSITIONAL.exec(ip);
+  if(transitionalMatch){
+    self.parseIPv4(transitionalMatch[1]);
+    return;
+  }
+
+  var bits = ip.split(':');
+  if (bits.length < 8) {
+    ip = ip.replace('::', Array(11 - bits.length).join(':'));
+    bits = ip.split(':');
+  }
+
+  var j = 0;
+  for (var i = 0; i < bits.length; i += 1) {
+    var x = bits[i] ? parseInt(bits[i], 16) : 0;
+    self.address[j++] = x >> 8;
+    self.address[j++] = x & 0xff;
+  }
+};
+
+IPCheck.prototype.match = function(cidr) {
+  var self = this;
+
+  if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
+  if (!self.valid || !cidr.valid) return false;
+
+  var mask = cidr.mask;
+  var i = 0;
+
+  while (mask >= 8) {
+    if (self.address[i] !== cidr.address[i]) return false;
+
+    i++;
+    mask -= 8;
+  }
+
+  var shift = 8 - mask;
+  return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
+};
+
+
+IPCheck.match = function(ip, cidr) {
+  ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
+  return ip.match(cidr);
+};

backend/src/utils/pairing-heap.ts

@@ -0,0 +1,174 @@
+export type HeapNode<T> = {
+  element: T
+  child?: HeapNode<T>
+  next?: HeapNode<T>
+  prev?: HeapNode<T>
+} | null | undefined;
+
+// minimal pairing heap priority queue implementation
+export class PairingHeap<T> {
+  private root: HeapNode<T> = null;
+  private comparator: (a: T, b: T) => boolean;
+
+  // comparator function should return 'true' if a is higher priority than b
+  constructor(comparator: (a: T, b: T) => boolean) {
+    this.comparator = comparator;
+  }
+
+  isEmpty(): boolean {
+    return !this.root;
+  }
+
+  add(element: T): HeapNode<T> {
+    const node: HeapNode<T> = {
+      element
+    };
+
+    this.root = this.meld(this.root, node);
+
+    return node;
+  }
+
+  // returns the top priority element without modifying the queue
+  peek(): T | void {
+    return this.root?.element;
+  }
+
+  // removes and returns the top priority element
+  pop(): T | void {
+    let element;
+    if (this.root) {
+      const node = this.root;
+      element = node.element;
+      this.root = this.mergePairs(node.child);
+    }
+    return element;
+  }
+
+  deleteNode(node: HeapNode<T>): void {
+    if (!node) {
+      return;
+    }
+
+    if (node === this.root) {
+      this.root = this.mergePairs(node.child);
+    }
+    else {
+      if (node.prev) {
+        if (node.prev.child === node) {
+          node.prev.child = node.next;
+        }
+        else {
+          node.prev.next = node.next;
+        }
+      }
+      if (node.next) {
+        node.next.prev = node.prev;
+      }
+      this.root = this.meld(this.root, this.mergePairs(node.child));
+    }
+
+    node.child = null;
+    node.prev = null;
+    node.next = null;
+  }
+
+  // fix the heap after increasing the priority of a given node
+  increasePriority(node: HeapNode<T>): void {
+    // already the top priority element
+    if (!node || node === this.root) {
+      return;
+    }
+    // extract from siblings
+    if (node.prev) {
+      if (node.prev?.child === node) {
+        if (this.comparator(node.prev.element, node.element)) {
+          // already in a valid position
+          return;
+        }
+        node.prev.child = node.next;
+      }
+      else {
+        node.prev.next = node.next;
+      }
+    }
+    if (node.next) {
+      node.next.prev = node.prev;
+    }
+
+    this.root = this.meld(this.root, node);
+  }
+
+  decreasePriority(node: HeapNode<T>): void {
+    this.deleteNode(node);
+    this.root = this.meld(this.root, node);
+  }
+
+  meld(a: HeapNode<T>, b: HeapNode<T>): HeapNode<T> {
+    if (!a) {
+      return b;
+    }
+    if (!b || a === b) {
+      return a;
+    }
+
+    let parent: HeapNode<T> = b;
+    let child: HeapNode<T> = a;
+    if (this.comparator(a.element, b.element)) {
+      parent = a;
+      child = b;
+    }
+
+    child.next = parent.child;
+    if (parent.child) {
+      parent.child.prev = child;
+    }
+    child.prev = parent;
+    parent.child = child;
+
+    parent.next = null;
+    parent.prev = null;
+
+    return parent;
+  }
+
+  mergePairs(node: HeapNode<T>): HeapNode<T> {
+    if (!node) {
+      return null;
+    }
+
+    let current: HeapNode<T> = node;
+    let next: HeapNode<T>;
+    let nextCurrent: HeapNode<T>;
+    let pairs: HeapNode<T>;
+    let melded: HeapNode<T>;
+    while (current) {
+      next = current.next;
+      if (next) {
+        nextCurrent = next.next;
+        melded = this.meld(current, next);
+        if (melded) {
+          melded.prev = pairs;
+        }
+        pairs = melded;
+      }
+      else {
+        nextCurrent = null;
+        current.prev = pairs;
+        pairs = current;
+        break;
+      }
+      current = nextCurrent;
+    }
+
+    melded = null;
+    let prev: HeapNode<T>;
+    while (pairs) {
+      prev = pairs.prev;
+      melded = this.meld(melded, pairs);
+      pairs = prev;
+    }
+
+    return melded;
+  }
+}

backend/testSetup.ts

@@ -0,0 +1,5 @@
+jest.mock('./mempool-config.json', () => ({}), { virtual: true });
+jest.mock('./src/logger.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true });

backend/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig",
+  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"],
+  "compilerOptions": {
+    "types": ["node"]
+  },
+
+}