Correcting docker disk cache config variable

Update docker configs

.github/CODEOWNERS

@@ -0,0 +1 @@
+backend/src/api/database-migration.ts	@wiz @softsimon

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: ['https://mempool.space/sponsor'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/dependabot.yml

@@ -1,20 +1,47 @@
 version: 2
 updates:
-- package-ecosystem: npm
-  directory: "/backend"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-- package-ecosystem: npm
-  directory: "/frontend"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-- package-ecosystem: docker
-  directory: "/docker/backend"
-  schedule:
-    interval: weekly
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-      interval: "weekly"
+  - package-ecosystem: npm
+    directory: "/backend"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+    allow:
+      - dependency-type: "production"
+
+  - package-ecosystem: npm
+    directory: "/frontend"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+    allow:
+      - dependency-type: "production"
+
+  - package-ecosystem: docker
+    directory: "/docker/backend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: docker
+    directory: "/docker/frontend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.16.0", "18.5.0"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"
@@ -42,8 +42,10 @@ jobs:
         run: npm run lint
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
-      #  - name: Test
-      #    run: npm run test
+      - name: Unit Tests
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run test
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Build
         run: npm run build
@@ -53,7 +55,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.15.0", "18.5.0"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"

.github/workflows/cypress.yml

@@ -1,8 +1,11 @@
 name: Cypress Tests
 
 on:
+  push:
+    branches: [master]
   pull_request:
-    types: [ opened, review_requested, synchronize ]
+    types: [opened, synchronize]
+
 jobs:
   cypress:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
@@ -24,36 +27,36 @@ jobs:
           - module: "bisq"
             spec: |
               cypress/e2e/bisq/bisq.spec.ts
-          
+
     name: E2E tests for ${{ matrix.module }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: ${{ matrix.module }}
-          
+
       - name: Setup node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: 16.15.0
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 
       - name: Chrome browser tests (${{ matrix.module }})
-        uses: cypress-io/github-action@v4
+        uses: cypress-io/github-action@v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
           build: npm run config:defaults:${{ matrix.module }}
           start: npm run start:local-staging
-          wait-on: 'http://localhost:4200'
+          wait-on: "http://localhost:4200"
           wait-on-timeout: 120
           record: true
           parallel: true
           spec: ${{ matrix.spec }}
           group: Tests on Chrome (${{ matrix.module }})
           browser: "chrome"
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
+          ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
         env:
           COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}

.github/workflows/get_image_digest.yml

@@ -0,0 +1,26 @@
+name: 'Print images digest'
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Image Version'
+        required: false
+        default: 'latest'
+        type: string 
+jobs:
+  print-images-sha:
+    runs-on: 'ubuntu-latest'
+    name: Print digest for images
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: digest
+
+      - name: Run script
+        working-directory: digest
+        run: |
+          sh ./docker/scripts/get_image_digest.sh $VERSION
+        env: 
+          VERSION: ${{ github.event.inputs.version }}

.github/workflows/on-tag.yml

@@ -1,7 +1,7 @@
 name: Docker build on tag
 env:
   DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
   DOCKER_BUILDKIT: 0
   COMPOSE_DOCKER_CLI_BUILD: 0
 
@@ -21,16 +21,46 @@ jobs:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
     name: Build and push to DockerHub
     steps:
+      # Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
+      - name: Replace the current swap file
+        shell: bash
+        run: |
+          sudo swapoff /mnt/swapfile
+          sudo rm -v /mnt/swapfile
+          sudo fallocate -l 13G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+
+      - name: Show current memory and swap status
+        shell: bash
+        run: |
+          sudo free -h
+          echo
+          sudo swapon --show
+
+      - name: Mount a tmpfs over /var/lib/docker
+        shell: bash
+        run: |
+          if [ ! -d "/var/lib/docker" ]; then
+            echo "Directory '/var/lib/docker' not found"
+            exit 1
+          fi
+          sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
+          sudo systemctl restart docker
+          sudo df -h | grep docker
+
       - name: Set env variables
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Show set environment variables
         run: |
           printf "      TAG: %s\n"  "$TAG"
-      
+
       - name: Add SHORT_SHA env property with commit short sha
         run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
 
@@ -38,24 +68,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+        uses: actions/checkout@v3
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
+        uses: docker/setup-qemu-action@v2
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
+        uses: docker/setup-buildx-action@v2
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@661fd3eb7f2f20d8c7c84bc2b0509efd7a826628 # v2
+        uses: actions/cache@v3
         id: cache
         with:
           path: /tmp/.buildx-cache

.gitignore

@@ -3,3 +3,5 @@ data
 docker-compose.yml
 backend/mempool-config.json
 *.swp
+frontend/src/resources/config.template.js
+frontend/src/resources/config.js

.vscode/settings.json

@@ -1,4 +1,5 @@
 {
   "editor.tabSize": 2,
+  "typescript.preferences.importModuleSpecifier": "relative",
   "typescript.tsdk": "./backend/node_modules/typescript/lib"
 }

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project
-Copyright (c) 2019-2022 The Mempool Open Source Project Developers
+Copyright (c) 2019-2023 The Mempool Open Source Project Developers
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of (at your option) either:

README.md

@@ -1,5 +1,7 @@
 # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
+https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4
+
 Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
 It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.

backend/.eslintrc

@@ -31,6 +31,7 @@
     "prefer-const": 1,
     "prefer-rest-params": 1,
     "quotes": [1, "single", { "allowTemplateLiterals": true }],
-    "semi": 1
+    "semi": 1,
+    "eqeqeq": 1
   }
 }

backend/.gitignore

@@ -1,7 +1,8 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # production config and external assets
-
+!mempool-config.template.json
+!mempool-config.sample.json
 mempool-config.json
 pools.json
 icons.json

backend/README.md

@@ -110,6 +110,11 @@ Run the Mempool backend:
 
 ```
 npm run start
+
+```
+You can also set env var `MEMPOOL_CONFIG_FILE` to specify a custom config file location:
+```
+MEMPOOL_CONFIG_FILE=/path/to/mempool-config.json npm run start
 ```
 
 When it's running, you should see output like this:
@@ -155,7 +160,7 @@ npm install -g ts-node nodemon
 Then, run the watcher:
 
 ```
-nodemon src/index.ts --ignore cache/ --ignore pools.json
+nodemon src/index.ts --ignore cache/
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -166,50 +171,84 @@ Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe
 
 Run bitcoind on regtest:
    ```
-   bitcoind -regtest -rpcport=8332
+   bitcoind -regtest
    ```
 
 Create a new wallet, if needed:
    ```
-   bitcoin-cli -regtest -rpcport=8332 createwallet test
+   bitcoin-cli -regtest createwallet test
    ```
 
 Load wallet (this command may take a while if you have lot of UTXOs):
    ```
-   bitcoin-cli -regtest -rpcport=8332 loadwallet test
+   bitcoin-cli -regtest loadwallet test
    ```
 
 Get a new address:
    ```
-   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   address=$(bitcoin-cli -regtest getnewaddress)
    ```
 
 Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
    ```
-   bitcoin-cli -regtest -rpcport=8332 generatetoaddress 101 $address
+   bitcoin-cli -regtest generatetoaddress 101 $address
    ```
 
 Send 0.1 BTC at 5 sat/vB to another address:
    ```
-   ./src/bitcoin-cli -named -regtest -rpcport=8332 sendtoaddress address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress) amount=0.1 fee_rate=5
+   bitcoin-cli -named -regtest sendtoaddress address=$(bitcoin-cli -regtest getnewaddress) amount=0.1 fee_rate=5
    ```
 
 See more example of `sendtoaddress`:
    ```
-   ./src/bitcoin-cli sendtoaddress # will print the help
+   bitcoin-cli sendtoaddress # will print the help
    ```
 
-Mini script to generate transactions with random TX fee-rate (between 1 to 100 sat/vB). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
+Mini script to generate random network activity (random TX count with random tx fee-rate). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
    ```
    #!/bin/bash
-   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   address=$(bitcoin-cli -regtest getnewaddress)
+   bitcoin-cli -regtest generatetoaddress 101 $address
    for i in {1..1000000}
    do
-     ./src/bitcoin-cli -regtest -rpcport=8332 -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1  1 100)
+      for y in $(seq 1 "$(jot -r 1 1 1000)")
+      do
+         bitcoin-cli -regtest -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1 1 100)
+      done
+      bitcoin-cli -regtest generatetoaddress 1 $address
+      sleep 5
    done
    ```
 
 Generate block at regular interval (every 10 seconds in this example):
    ```
-   watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
+   watch -n 10 "bitcoin-cli -regtest generatetoaddress 1 $address"
    ```
+
+### Mining pools update
+
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
+
+To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
+
+You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+
+When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+
+### Re-index tables
+
+You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
+
+Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
+
+Usage:
+```
+npm run start --reindex=blocks,hashrates
+```
+Example output:
+```
+Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
+Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
+```
+
+Reference: https://github.com/mempool/mempool/pull/1269

backend/jest.config.ts

@@ -0,0 +1,20 @@
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  verbose: true,
+  automock: false,
+  collectCoverage: true,
+  collectCoverageFrom: ["./src/**/**.ts"],
+  coverageProvider: "babel",
+  coverageThreshold: {
+    global: {
+      lines: 1
+    }
+  },
+  setupFiles: [
+    "./testSetup.ts",
+  ],
+}
+export default config;

backend/mempool-config.sample.json

@@ -2,6 +2,7 @@
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
@@ -14,14 +15,20 @@
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "INDEXING_BLOCKS_AMOUNT": 11000,
     "BLOCKS_SUMMARIES_INDEXING": false,
-    "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": [],
     "EXTERNAL_MAX_RETRY": 1,
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
-    "AUTOMATIC_BLOCK_REINDEXING": false
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "AUDIT": false,
+    "ADVANCED_GBT_AUDIT": false,
+    "ADVANCED_GBT_MEMPOOL": false,
+    "CPFP_INDEXING": false,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -75,12 +82,20 @@
   },
   "LIGHTNING": {
     "ENABLED": false,
-    "BACKEND": "lnd"
+    "BACKEND": "lnd",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": 20
   },
   "LND": {
     "TLS_CERT_PATH": "tls.cert",
-    "MACAROON_PATH": "admin.macaroon",
-    "SOCKET": "localhost:10009"
+    "MACAROON_PATH": "readonly.macaroon",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "lightning-rpc"
   },
   "SOCKS5PROXY": {
     "ENABLED": false,

backend/package.json

@@ -16,44 +16,53 @@
     "mempool",
     "blockchain",
     "explorer",
-    "liquid"
+    "liquid",
+    "lightning"
   ],
   "main": "index.ts",
   "scripts": {
-    "tsc": "./node_modules/typescript/bin/tsc",
-    "build": "npm run tsc",
+    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
+    "build": "npm run tsc && npm run create-resources",
+    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
+    "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "start-production": "node --max-old-space-size=16384 dist/index.js",
+    "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
+    "@babel/core": "^7.20.12",
     "@mempool/electrum-client": "^1.1.7",
-    "@types/node": "^16.11.41",
+    "@types/node": "^16.18.11",
     "axios": "~0.27.2",
-    "bitcoinjs-lib": "6.0.1",
-    "bolt07": "^1.8.1",
-    "crypto-js": "^4.0.0",
-    "express": "^4.18.0",
-    "lightning": "^5.16.3",
-    "maxmind": "^4.3.6",
-    "mysql2": "2.3.3",
-    "node-worker-threads-pool": "^1.5.1",
+    "bitcoinjs-lib": "~6.1.0",
+    "crypto-js": "~4.1.1",
+    "express": "~4.18.2",
+    "maxmind": "~4.3.8",
+    "mysql2": "~2.3.3",
+    "node-worker-threads-pool": "~1.5.1",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.7.4",
-    "ws": "~8.8.0"
+    "ws": "~8.11.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.20.7",
+    "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
-    "@types/express": "^4.17.13",
-    "@types/ws": "~8.5.3",
-    "@typescript-eslint/eslint-plugin": "^5.30.5",
-    "@typescript-eslint/parser": "^5.30.5",
-    "eslint": "^8.19.0",
+    "@types/express": "^4.17.15",
+    "@types/jest": "^29.2.5",
+    "@types/ws": "~8.5.4",
+    "@typescript-eslint/eslint-plugin": "^5.48.1",
+    "@typescript-eslint/parser": "^5.48.1",
+    "eslint": "^8.31.0",
     "eslint-config-prettier": "^8.5.0",
-    "prettier": "^2.7.1"
+    "jest": "^29.3.1",
+    "prettier": "^2.8.2",
+    "ts-jest": "^29.0.3",
+    "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -0,0 +1,116 @@
+{
+  "MEMPOOL": {
+    "ENABLED": true,
+    "NETWORK": "__MEMPOOL_NETWORK__",
+    "BACKEND": "__MEMPOOL_BACKEND__",
+    "BLOCKS_SUMMARIES_INDEXING": true,
+    "HTTP_PORT": 1,
+    "SPAWN_CLUSTER_PROCS": 2,
+    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POLL_RATE_MS": 3,
+    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CLEAR_PROTECTION_MINUTES": 4,
+    "RECOMMENDED_FEE_PERCENTILE": 5,
+    "BLOCK_WEIGHT_UNITS": 6,
+    "INITIAL_BLOCKS_AMOUNT": 7,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "USE_SECOND_NODE_FOR_MINFEE": 10,
+    "EXTERNAL_ASSETS": 11,
+    "EXTERNAL_MAX_RETRY": 12,
+    "EXTERNAL_RETRY_INTERVAL": 13,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
+    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
+    "INDEXING_BLOCKS_AMOUNT": 14,
+    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "AUDIT": "__MEMPOOL_AUDIT__",
+    "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
+    "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
+    "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
+  },
+  "CORE_RPC": {
+    "HOST": "__CORE_RPC_HOST__",
+    "PORT": 15,
+    "USERNAME": "__CORE_RPC_USERNAME__",
+    "PASSWORD": "__CORE_RPC_PASSWORD__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRUM_HOST__",
+    "PORT": 16,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "__SECOND_CORE_RPC_HOST__",
+    "PORT": 17,
+    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+  },
+  "DATABASE": {
+    "ENABLED": false,
+    "HOST": "__DATABASE_HOST__",
+    "SOCKET": "__DATABASE_SOCKET__",
+    "PORT": 18,
+    "DATABASE": "__DATABASE_DATABASE__",
+    "USERNAME": "__DATABASE_USERNAME__",
+    "PASSWORD": "__DATABASE_PASSWORD__"
+  },
+  "SYSLOG": {
+    "ENABLED": false,
+    "HOST": "__SYSLOG_HOST__",
+    "PORT": 19,
+    "MIN_PRIORITY": "__SYSLOG_MIN_PRIORITY__",
+    "FACILITY": "__SYSLOG_FACILITY__"
+  },
+  "STATISTICS": {
+    "ENABLED": false,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 20
+  },
+  "BISQ": {
+    "ENABLED": true,
+    "DATA_PATH": "__BISQ_DATA_PATH__"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": true,
+    "USE_ONION": true,
+    "HOST": "__SOCKS5PROXY_HOST__",
+    "PORT": "__SOCKS5PROXY_PORT__",
+    "USERNAME": "__SOCKS5PROXY_USERNAME__",
+    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
+    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
+  },
+  "LIGHTNING": {
+    "ENABLED": "__LIGHTNING_ENABLED__",
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__"
+  },
+  "LND": {
+    "TLS_CERT_PATH": "",
+    "MACAROON_PATH": "",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  }
+}

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -0,0 +1,66 @@
+import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+
+describe('Mempool Difficulty Adjustment', () => {
+  test('should calculate Difficulty Adjustments properly', () => {
+    const dt = (dtString) => {
+      return Math.floor(new Date(dtString).getTime() / 1000);
+    };
+
+    const vectors = [
+      [ // Vector 1
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // If not testnet, not used
+        ],
+        { // Expected Result
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          previousTime: 1660820820,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: 0,
+          expectedBlocks: 161.68833333333333,
+        },
+      ],
+      [ // Vector 2 (testnet)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'testnet',                      // Network
+          dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result is same other than timeOffset
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousTime: 1660820820,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
+          // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
+          expectedBlocks: 161.68833333333333,
+        },
+      ],
+    ] as [[number, number, number, number, string, number], DifficultyAdjustment][];
+
+    for (const vector of vectors) {
+      const result = calcDifficultyAdjustment(...vector[0]);
+      // previousRetarget is passed through untouched
+      expect(result.previousRetarget).toStrictEqual(vector[0][3]);
+      expect(result).toStrictEqual(vector[1]);
+    }
+  });
+});

backend/src/__tests__/config.test.ts

@@ -0,0 +1,152 @@
+import * as fs from 'fs';
+
+describe('Mempool Backend Config', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  test('should return defaults when no file is present', () => {
+    jest.isolateModules(() => {
+      jest.mock('../../mempool-config.json', () => ({}), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual({
+        ENABLED: true,
+        NETWORK: 'mainnet',
+        BACKEND: 'none',
+        BLOCKS_SUMMARIES_INDEXING: false,
+        HTTP_PORT: 8999,
+        SPAWN_CLUSTER_PROCS: 0,
+        API_URL_PREFIX: '/api/v1/',
+        AUTOMATIC_BLOCK_REINDEXING: false,
+        POLL_RATE_MS: 2000,
+        CACHE_DIR: './cache',
+        CLEAR_PROTECTION_MINUTES: 20,
+        RECOMMENDED_FEE_PERCENTILE: 50,
+        BLOCK_WEIGHT_UNITS: 4000000,
+        INITIAL_BLOCKS_AMOUNT: 8,
+        MEMPOOL_BLOCKS_AMOUNT: 8,
+        INDEXING_BLOCKS_AMOUNT: 11000,
+        USE_SECOND_NODE_FOR_MINFEE: false,
+        EXTERNAL_ASSETS: [],
+        EXTERNAL_MAX_RETRY: 1,
+        EXTERNAL_RETRY_INTERVAL: 0,
+        USER_AGENT: 'mempool',
+        STDOUT_LOG_MIN_PRIORITY: 'debug',
+        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+        AUDIT: false,
+        ADVANCED_GBT_AUDIT: false,
+        ADVANCED_GBT_MEMPOOL: false,
+        CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
+        DISK_CACHE_BLOCK_INTERVAL: 6,
+      });
+
+      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
+
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+
+      expect(config.CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.DATABASE).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        SOCKET: '',
+        PORT: 3306,
+        DATABASE: 'mempool',
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SYSLOG).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        PORT: 514,
+        MIN_PRIORITY: 'info',
+        FACILITY: 'local7'
+      });
+
+      expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
+
+      expect(config.BISQ).toStrictEqual({ ENABLED: false, DATA_PATH: '/bisq/statsnode-data/btc_mainnet/db' });
+
+      expect(config.SOCKS5PROXY).toStrictEqual({
+        ENABLED: false,
+        USE_ONION: true,
+        HOST: '127.0.0.1',
+        PORT: 9050,
+        USERNAME: '',
+        PASSWORD: ''
+      });
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual({
+        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
+      });
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
+        MEMPOOL_API: 'https://mempool.space/api/v1',
+        MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+        LIQUID_API: 'https://liquid.network/api/v1',
+        LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+        BISQ_URL: 'https://bisq.markets/api',
+        BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+      });
+
+      expect(config.MAXMIND).toStrictEqual({
+        ENABLED: false,
+        GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
+        GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
+        GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
+      });
+    });
+  });
+
+  test('should override the default values with the passed values', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      jest.mock('../../mempool-config.json', () => (fixture), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual(fixture.MEMPOOL);
+
+      expect(config.ELECTRUM).toStrictEqual(fixture.ELECTRUM);
+
+      expect(config.ESPLORA).toStrictEqual(fixture.ESPLORA);
+
+      expect(config.CORE_RPC).toStrictEqual(fixture.CORE_RPC);
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual(fixture.SECOND_CORE_RPC);
+
+      expect(config.DATABASE).toStrictEqual(fixture.DATABASE);
+
+      expect(config.SYSLOG).toStrictEqual(fixture.SYSLOG);
+
+      expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
+
+      expect(config.BISQ).toStrictEqual(fixture.BISQ);
+
+      expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+    });
+  });
+});

backend/src/api/audit.ts

@@ -0,0 +1,143 @@
+import config from '../config';
+import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+
+const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
+
+class Audit {
+  auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
+   : { censored: string[], added: string[], fresh: string[], score: number, similarity: number } {
+    if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
+      return { censored: [], added: [], fresh: [], score: 0, similarity: 1 };
+    }
+
+    const matches: string[] = []; // present in both mined block and template
+    const added: string[] = []; // present in mined block, not in template
+    const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const isCensored = {}; // missing, without excuse
+    const isDisplaced = {};
+    let displacedWeight = 0;
+    let matchedWeight = 0;
+    let projectedWeight = 0;
+
+    const inBlock = {};
+    const inTemplate = {};
+
+    const now = Math.round((Date.now() / 1000));
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+    // coinbase is always expected
+    if (transactions[0]) {
+      inTemplate[transactions[0].txid] = true;
+    }
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const txid of projectedBlocks[0].transactionIds) {
+      if (!inBlock[txid]) {
+        // tx is recent, may have reached the miner too late for inclusion
+        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          fresh.push(txid);
+        } else {
+          isCensored[txid] = true;
+        }
+        displacedWeight += mempool[txid].weight;
+      } else {
+        matchedWeight += mempool[txid].weight;
+      }
+      projectedWeight += mempool[txid].weight;
+      inTemplate[txid] = true;
+    }
+
+    displacedWeight += (4000 - transactions[0].weight);
+    projectedWeight += transactions[0].weight;
+    matchedWeight += transactions[0].weight;
+
+    // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
+    // these displaced transactions should occupy the first N weight units of the next projected block
+    let displacedWeightRemaining = displacedWeight;
+    let index = 0;
+    let lastFeeRate = Infinity;
+    let failures = 0;
+    while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
+      const txid = projectedBlocks[1].transactionIds[index];
+      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (fits || feeMatches) {
+        isDisplaced[txid] = true;
+        if (fits) {
+          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+        }
+        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+          displacedWeightRemaining -= mempool[txid].weight;
+        }
+        failures = 0;
+      } else {
+        failures++;
+      }
+      index++;
+    }
+
+    // mark unexpected transactions in the mined block as 'added'
+    let overflowWeight = 0;
+    let totalWeight = 0;
+    for (const tx of transactions) {
+      if (inTemplate[tx.txid]) {
+        matches.push(tx.txid);
+      } else {
+        if (!isDisplaced[tx.txid]) {
+          added.push(tx.txid);
+        } else {
+        }
+        let blockIndex = -1;
+        let index = -1;
+        projectedBlocks.forEach((block, bi) => {
+          const i = block.transactionIds.indexOf(tx.txid);
+          if (i >= 0) {
+            blockIndex = bi;
+            index = i;
+          }
+        });
+        overflowWeight += tx.weight;
+      }
+      totalWeight += tx.weight;
+    }
+
+    // transactions missing from near the end of our template are probably not being censored
+    let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
+    let maxOverflowRate = 0;
+    let rateThreshold = 0;
+    index = projectedBlocks[0].transactionIds.length - 1;
+    while (index >= 0) {
+      const txid = projectedBlocks[0].transactionIds[index];
+      if (overflowWeightRemaining > 0) {
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
+          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+        }
+      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+      }
+      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      index--;
+    }
+
+    const numCensored = Object.keys(isCensored).length;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
+    const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
+
+    return {
+      censored: Object.keys(isCensored),
+      added,
+      fresh,
+      score,
+      similarity,
+    };
+  }
+}
+
+export default new Audit();

backend/src/api/backend-info.ts

@@ -1,60 +1,39 @@
-import * as fs from 'fs';
-import * as os from 'os';
-import logger from '../logger';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
-const { spawnSync } = require('child_process');
+import config from '../config';
 
 class BackendInfo {
-  private gitCommitHash = '';
-  private hostname = '';
-  private version = '';
+  private backendInfo: IBackendInfo;
 
   constructor() {
-    this.setLatestCommitHash();
-    this.setVersion();
-    this.hostname = os.hostname();
-  }
-
-  public getBackendInfo(): IBackendInfo {
-    return {
-      hostname: this.hostname,
-      gitCommit: this.gitCommitHash,
-      version: this.version,
+    // This file is created by ./fetch-version.ts during building
+    const versionFile = path.join(__dirname, 'version.json')
+    var versionInfo;
+    if (fs.existsSync(versionFile)) {
+      versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
+    } else {
+      // Use dummy values if `versionFile` doesn't exist (e.g., during testing)
+      versionInfo = {
+        version: '?',
+        gitCommit: '?'
+      };
+    }
+    this.backendInfo = {
+      hostname: os.hostname(),
+      version: versionInfo.version,
+      gitCommit: versionInfo.gitCommit,
+      lightning: config.LIGHTNING.ENABLED
     };
   }
 
+  public getBackendInfo(): IBackendInfo {
+    return this.backendInfo;
+  }
+
   public getShortCommitHash() {
-    return this.gitCommitHash.slice(0, 7);
-  }
-
-  private setLatestCommitHash(): void {
-    //TODO: share this logic with `generate-config.js`
-    if (process.env.DOCKER_COMMIT_HASH) {
-      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
-    } else {
-      try {
-        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
-        if (!gitRevParse.error) {
-          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
-          this.gitCommitHash = output ? output : '?';
-        } else if (gitRevParse.error.code === 'ENOENT') {
-          console.log('git not found, cannot parse git hash');
-          this.gitCommitHash = '?';
-        }
-      } catch (e: any) {
-        console.log('Could not load git commit info: ' + e.message);
-        this.gitCommitHash = '?';
-      }
-    }
-  }
-
-  private setVersion(): void {
-    try {
-      const packageJson = fs.readFileSync('package.json').toString();
-      this.version = JSON.parse(packageJson).version;
-    } catch (e) {
-      throw new Error(e instanceof Error ? e.message : 'Error');
-    }
+    return this.backendInfo.gitCommit.slice(0, 7);
   }
 }
 

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -3,13 +3,14 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getTransactionHex(txId: string): Promise<string>;
   $getBlockHeightTip(): Promise<number>;
   $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
-  $getRawBlock(hash: string): Promise<string>;
+  $getRawBlock(hash: string): Promise<Buffer>;
   $getAddress(address: string): Promise<IEsploraApi.Address>;
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];

backend/src/api/bitcoin/bitcoin-api-factory.ts

@@ -17,4 +17,6 @@ function bitcoinApiFactory(): AbstractBitcoinApi {
   }
 }
 
+export const bitcoinCoreApi = new BitcoinApi(bitcoinClient);
+
 export default bitcoinApiFactory();

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -172,4 +172,35 @@ export namespace IBitcoinApi {
     }
   }
 
+  export interface BlockStats {
+    "avgfee": number;
+    "avgfeerate": number;
+    "avgtxsize": number;
+    "blockhash": string;
+    "feerate_percentiles": [number, number, number, number, number];
+    "height": number;
+    "ins": number;
+    "maxfee": number;
+    "maxfeerate": number;
+    "maxtxsize": number;
+    "medianfee": number;
+    "mediantime": number;
+    "mediantxsize": number;
+    "minfee": number;
+    "minfeerate": number;
+    "mintxsize": number;
+    "outs": number;
+    "subsidy": number;
+    "swtotal_size": number;
+    "swtotal_weight": number;
+    "swtxs": number;
+    "time": number;
+    "total_out": number;
+    "total_size": number;
+    "total_weight": number;
+    "totalfee": number;
+    "txs": number;
+    "utxo_increase": number;
+    "utxo_size_inc": number;
+  }
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       size: block.size,
       weight: block.weight,
       previousblockhash: block.previousblockhash,
+      mediantime: block.mediantime,
     };
   }
 
@@ -57,6 +58,11 @@ class BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
+  $getTransactionHex(txId: string): Promise<string> {
+    return this.$getRawTransaction(txId, true)
+      .then((tx) => tx.hex || '');
+  }
+
   $getBlockHeightTip(): Promise<number> {
     return this.bitcoindClient.getChainTips()
       .then((result: IBitcoinApi.ChainTips[]) => {
@@ -76,7 +82,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
-  $getRawBlock(hash: string): Promise<string> {
+  $getRawBlock(hash: string): Promise<Buffer> {
     return this.bitcoindClient.getBlock(hash, 0)
       .then((raw: string) => Buffer.from(raw, "hex"));
   }

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -1,11 +1,12 @@
 import { Application, Request, Response } from 'express';
 import axios from 'axios';
+import * as bitcoinjs from 'bitcoinjs-lib';
 import config from '../../config';
 import websocketHandler from '../websocket-handler';
 import mempool from '../mempool';
 import feeApi from '../fee-api';
 import mempoolBlocks from '../mempool-blocks';
-import bitcoinApi from './bitcoin-api-factory';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
 import { Common } from '../common';
 import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
@@ -16,19 +17,23 @@ import logger from '../../logger';
 import blocks from '../blocks';
 import bitcoinClient from './bitcoin-client';
 import difficultyAdjustment from '../difficulty-adjustment';
+import transactionRepository from '../../repositories/TransactionRepository';
+import rbfCache from '../rbf-cache';
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
     app
       .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.getCpfpInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
       .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
@@ -87,7 +92,11 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', this.getBlocks.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions);
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -185,29 +194,43 @@ class BitcoinRoutes {
     }
   }
 
-  private getCpfpInfo(req: Request, res: Response) {
+  private async $getCpfpInfo(req: Request, res: Response) {
     if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
       res.status(501).send(`Invalid transaction ID.`);
       return;
     }
 
     const tx = mempool.getMempool()[req.params.txId];
-    if (!tx) {
-      res.status(404).send(`Transaction doesn't exist in the mempool.`);
+    if (tx) {
+      if (tx?.cpfpChecked) {
+        res.json({
+          ancestors: tx.ancestors,
+          bestDescendant: tx.bestDescendant || null,
+          descendants: tx.descendants || null,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+        });
+        return;
+      }
+
+      const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+
+      res.json(cpfpInfo);
       return;
+    } else {
+      let cpfpInfo;
+      if (config.DATABASE.ENABLED) {
+        cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      }
+      if (cpfpInfo) {
+        res.json(cpfpInfo);
+        return;
+      } else {
+        res.json({
+          ancestors: []
+        });
+        return;
+      }
     }
-
-    if (tx.cpfpChecked) {
-      res.json({
-        ancestors: tx.ancestors,
-        bestDescendant: tx.bestDescendant || null,
-      });
-      return;
-    }
-
-    const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
-
-    res.json(cpfpInfo);
   }
 
   private getBackendInfo(req: Request, res: Response) {
@@ -241,6 +264,74 @@ class BitcoinRoutes {
     }
   }
 
+  /**
+   * Takes the PSBT as text/plain body, parses it, and adds the full
+   * parent transaction to each input that doesn't already have it.
+   * This is used for BTCPayServer / Trezor users which need access to
+   * the full parent transaction even with segwit inputs.
+   * It will respond with a text/plain PSBT in the same format (hex|base64).
+   */
+  private async postPsbtCompletion(req: Request, res: Response): Promise<void> {
+    res.setHeader('content-type', 'text/plain');
+    const notFoundError = `Couldn't get transaction hex for parent of input`;
+    try {
+      let psbt: bitcoinjs.Psbt;
+      let format: 'hex' | 'base64';
+      let isModified = false;
+      try {
+        psbt = bitcoinjs.Psbt.fromBase64(req.body);
+        format = 'base64';
+      } catch (e1) {
+        try {
+          psbt = bitcoinjs.Psbt.fromHex(req.body);
+          format = 'hex';
+        } catch (e2) {
+          throw new Error(`Unable to parse PSBT`);
+        }
+      }
+      for (const [index, input] of psbt.data.inputs.entries()) {
+        if (!input.nonWitnessUtxo) {
+          // Buffer.from ensures it won't be modified in place by reverse()
+          const txid = Buffer.from(psbt.txInputs[index].hash)
+            .reverse()
+            .toString('hex');
+
+          let transactionHex: string;
+          // If missing transaction, return 404 status error
+          try {
+            transactionHex = await bitcoinApi.$getTransactionHex(txid);
+            if (!transactionHex) {
+              throw new Error('');
+            }
+          } catch (err) {
+            throw new Error(`${notFoundError} #${index} @ ${txid}`);
+          }
+
+          psbt.updateInput(index, {
+            nonWitnessUtxo: Buffer.from(transactionHex, 'hex'),
+          });
+          if (!isModified) {
+            isModified = true;
+          }
+        }
+      }
+      if (isModified) {
+        res.send(format === 'hex' ? psbt.toHex() : psbt.toBase64());
+      } else {
+        // Not modified
+        // 422 Unprocessable Entity
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
+        res.status(422).send(`Psbt had no missing nonWitnessUtxos.`);
+      }
+    } catch (e: any) {
+      if (e instanceof Error && new RegExp(notFoundError).test(e.message)) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
   private async getTransactionStatus(req: Request, res: Response) {
     try {
       const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
@@ -254,6 +345,16 @@ class BitcoinRoutes {
     }
   }
 
+  private async getStrippedBlockTransactions(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getBlock(req: Request, res: Response) {
     try {
       const block = await blocks.$getBlock(req.params.hash);
@@ -286,9 +387,9 @@ class BitcoinRoutes {
     }
   }
 
-  private async getStrippedBlockTransactions(req: Request, res: Response) {
+  private async getBlockAuditSummary(req: Request, res: Response) {
     try {
-      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      const transactions = await blocks.$getBlockAuditSummary(req.params.hash);
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
       res.json(transactions);
     } catch (e) {
@@ -310,10 +411,46 @@ class BitcoinRoutes {
     }
   }
 
+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!req.params.from || from < 0) {
+        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (to < 0) {
+        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+      }
+      if (from > to) {
+        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+      }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
+
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getLegacyBlocks(req: Request, res: Response) {
     try {
       const returnBlocks: IEsploraApi.Block[] = [];
-      const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();
+      const tip = blocks.getCurrentBlockHeight();
+      const fromHeight = Math.min(parseInt(req.params.height, 10) || tip, tip);
 
       // Check if block height exist in local cache to skip the hash lookup
       const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight);
@@ -331,7 +468,7 @@ class BitcoinRoutes {
           returnBlocks.push(localBlock);
           nextHash = localBlock.previousblockhash;
         } else {
-          const block = await bitcoinApi.$getBlock(nextHash);
+          const block = await bitcoinCoreApi.$getBlock(nextHash);
           returnBlocks.push(block);
           nextHash = block.previousblockhash;
         }
@@ -499,6 +636,28 @@ class BitcoinRoutes {
     }
   }
 
+  private async getRbfHistory(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getReplaces(req.params.txId);
+      res.json(result || []);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getCachedTx(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getTx(req.params.txId);
+      if (result) {
+        res.json(result);
+      } else {
+        res.status(204).send();
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getTransactionOutspends(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getOutspends(req.params.txId);
@@ -510,7 +669,12 @@ class BitcoinRoutes {
 
   private getDifficultyChange(req: Request, res: Response) {
     try {
-      res.json(difficultyAdjustment.getDifficultyAdjustment());
+      const da = difficultyAdjustment.getDifficultyAdjustment();
+      if (da) {
+        res.json(da);
+      } else {
+        res.status(503).send(`Service Temporarily Unavailable`);
+      }
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -88,6 +88,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    mediantime: number;
   }
 
   export interface Address {

backend/src/api/bitcoin/esplora-api.ts

@@ -1,8 +1,13 @@
 import config from '../../config';
 import axios, { AxiosRequestConfig } from 'axios';
+import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 
+const axiosConnection = axios.create({
+  httpAgent: new http.Agent({ keepAlive: true })
+});
+
 class ElectrsApi implements AbstractBitcoinApi {
   axiosConfig: AxiosRequestConfig = {
     timeout: 10000,
@@ -11,48 +16,53 @@ class ElectrsApi implements AbstractBitcoinApi {
   constructor() { }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getTransactionHex(txId: string): Promise<string> {
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHashTip(): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
       .then((response) => response.data);
   }
 
-  $getRawBlock(hash: string): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", this.axiosConfig)
-      .then((response) => response.data);
+  $getRawBlock(hash: string): Promise<Buffer> {
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+      .then((response) => { return Buffer.from(response.data); });
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {
@@ -72,12 +82,12 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
       .then((response) => response.data);
   }
 

backend/src/api/blocks.ts

@@ -1,8 +1,8 @@
 import config from '../config';
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -13,15 +13,18 @@ import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
 import loadingIndicators from './loading-indicators';
 import BitcoinApi from './bitcoin/bitcoin-api';
-import { prepareBlock } from '../utils/blocks-utils';
 import BlocksRepository from '../repositories/BlocksRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
-import fiatConversion from './fiat-conversion';
 import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import cpfpRepository from '../repositories/CpfpRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import PricesRepository from '../repositories/PricesRepository';
+import priceUpdater from '../tasks/price-updater';
+import chainTips from './chain-tips';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -31,6 +34,7 @@ class Blocks {
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
+  private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
   constructor() { }
 
@@ -54,6 +58,10 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
+  public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>) {
+    this.newAsyncBlockCallbacks.push(fn);
+  }
+
   /**
    * Return the list of transaction for a block
    * @param blockHash
@@ -90,12 +98,23 @@ class Blocks {
           transactions.push(tx);
           transactionsFetched++;
         } catch (e) {
-          if (i === 0) {
-            const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
-            logger.err(msg);
-            throw new Error(msg);
-          } else {
-            logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+          try {
+            if (config.MEMPOOL.BACKEND === 'esplora') {
+              // Try again with core
+              const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true);
+              transactions.push(tx);
+              transactionsFetched++;
+            } else {
+              throw e;
+            }
+          } catch (e) {
+            if (i === 0) {
+              const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
+              logger.err(msg);
+              throw new Error(msg);
+            } else {
+              logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+            }
           }
         }
       }
@@ -123,11 +142,14 @@ class Blocks {
    * @param block
    * @returns BlockSummary
    */
-  private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
-    const stripped = block.tx.map((tx) => {
+  public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
+    if (Common.isLiquid()) {
+      block = this.convertLiquidFees(block);
+    }
+    const stripped = block.tx.map((tx: IBitcoinApi.VerboseTransaction) => {
       return {
         txid: tx.txid,
-        vsize: tx.vsize,
+        vsize: tx.weight / 4,
         fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
         value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
       };
@@ -139,6 +161,13 @@ class Blocks {
     };
   }
 
+  private convertLiquidFees(block: IBitcoinApi.VerboseBlock): IBitcoinApi.VerboseBlock {
+    block.tx.forEach(tx => {
+      tx.fee = Object.values(tx.fee || {}).reduce((total, output) => total + output, 0);
+    });
+    return block;
+  }
+
   /**
    * Return a block with additional data (reward, coinbase, fees...)
    * @param block
@@ -146,33 +175,81 @@ class Blocks {
    * @returns BlockExtended
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
-    blockExtended.extras.usd = fiatConversion.getConversionRates().USD;
+    const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    
+    const blk: Partial<BlockExtended> = Object.assign({}, block);
+    const extras: Partial<BlockExtension> = {};
+
+    extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
+    extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
 
     if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
+      extras.medianFee = 0; // 50th percentiles
+      extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      extras.totalFees = 0;
+      extras.avgFee = 0;
+      extras.avgFeeRate = 0;
+      extras.utxoSetChange = 0;
+      extras.avgTxSize = 0;
+      extras.totalInputs = 0;
+      extras.totalOutputs = 1;
+      extras.totalOutputAmt = 0;
+      extras.segwitTotalTxs = 0;
+      extras.segwitTotalSize = 0;
+      extras.segwitTotalWeight = 0;
     } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
-      ]);
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.totalFees = stats.totalfee;
-      blockExtended.extras.avgFee = stats.avgfee;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+      const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
+      extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      extras.totalFees = stats.totalfee;
+      extras.avgFee = stats.avgfee;
+      extras.avgFeeRate = stats.avgfeerate;
+      extras.utxoSetChange = stats.utxo_increase;
+      extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
+      extras.totalInputs = stats.ins;
+      extras.totalOutputs = stats.outs;
+      extras.totalOutputAmt = stats.total_out;
+      extras.segwitTotalTxs = stats.swtxs;
+      extras.segwitTotalSize = stats.swtotal_size;
+      extras.segwitTotalWeight = stats.swtotal_weight;
+    }
+
+    if (Common.blocksSummariesIndexingEnabled()) {
+      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+      if (extras.feePercentiles !== null) {
+        extras.medianFeeAmt = extras.feePercentiles[3];
+      }
+    }
+  
+    extras.virtualSize = block.weight / 4.0;
+    if (coinbaseTx?.vout.length > 0) {
+      extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+      extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
+    } else {
+      extras.coinbaseAddress = null;
+      extras.coinbaseSignature = null;
+      extras.coinbaseSignatureAscii = null;
+    }
+
+    const header = await bitcoinClient.getBlockHeader(block.id, false);
+    extras.header = header;
+
+    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
+    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+      extras.utxoSetSize = txoutset.txouts,
+      extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
+    } else {
+      extras.utxoSetSize = null;
+      extras.totalInputAmt = null;
     }
 
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      if (coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
           pool = await poolsRepository.$getUnknownPool();
@@ -182,19 +259,27 @@ class Blocks {
       }
 
       if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
-        return blockExtended;
+      } else {
+        extras.pool = {
+          id: pool.uniqueId,
+          name: pool.name,
+          slug: pool.slug,
+        };
       }
 
-      blockExtended.extras.pool = {
-        id: pool.id,
-        name: pool.name,
-        slug: pool.slug,
-      };
+      extras.matchRate = null;
+      if (config.MEMPOOL.AUDIT) {
+        const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
+        if (auditScore != null) {
+          extras.matchRate = auditScore.matchRate;
+        }
+      }
     }
 
-    return blockExtended;
+    blk.extras = <BlockExtension>extras;
+    return <BlockExtended>blk;
   }
 
   /**
@@ -220,15 +305,18 @@ class Blocks {
     } else {
       pools = poolsParser.miningPools;
     }
+
     for (let i = 0; i < pools.length; ++i) {
       if (address !== undefined) {
-        const addresses: string[] = JSON.parse(pools[i].addresses);
+        const addresses: string[] = typeof pools[i].addresses === 'string' ?
+          JSON.parse(pools[i].addresses) : pools[i].addresses;
         if (addresses.indexOf(address) !== -1) {
           return pools[i];
         }
       }
 
-      const regexes: string[] = JSON.parse(pools[i].regexes);
+      const regexes: string[] = typeof pools[i].regexes === 'string' ?
+        JSON.parse(pools[i].regexes) : pools[i].regexes;
       for (let y = 0; y < regexes.length; ++y) {
         const regex = new RegExp(regexes[y], 'i');
         const match = asciiScriptSig.match(regex);
@@ -248,7 +336,7 @@ class Blocks {
   /**
    * [INDEXING] Index all blocks summaries for the block txs visualization
    */
-  public async $generateBlocksSummariesDatabase() {
+  public async $generateBlocksSummariesDatabase(): Promise<void> {
     if (Common.blocksSummariesIndexingEnabled() === false) {
       return;
     }
@@ -281,7 +369,7 @@ class Blocks {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
           const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
           const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
-          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
         }
@@ -294,12 +382,60 @@ class Blocks {
         newlyIndexed++;
       }
       if (newlyIndexed > 0) {
-        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       } else {
-        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       }
     } catch (e) {
-      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Index transaction CPFP data for all blocks
+   */
+   public async $generateCPFPDatabase(): Promise<void> {
+    if (Common.cpfpIndexingEnabled() === false) {
+      return;
+    }
+
+    try {
+      // Get all indexed block hash
+      const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
+      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
+
+      if (!unindexedBlockHeights?.length) {
+        return;
+      }
+
+      // Logging
+      let count = 0;
+      let countThisRun = 0;
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+      for (const height of unindexedBlockHeights) {
+        // Logging
+        const hash = await bitcoinApi.$getBlockHash(height);
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const blockPerSeconds = (countThisRun / elapsedSeconds);
+          const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
+          logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          timer = new Date().getTime() / 1000;
+          countThisRun = 0;
+        }
+
+        await this.$indexCPFP(hash, height); // Calculate and save CPFP data for transactions in this block
+
+        // Logging
+        count++;
+        countThisRun++;
+      }
+      logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
+    } catch (e) {
+      logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -319,7 +455,7 @@ class Blocks {
 
       const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
 
-      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
+      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`, logger.tags.mining);
       loadingIndicators.setProgress('block-indexing', 0);
 
       const chunkSize = 10000;
@@ -339,7 +475,7 @@ class Blocks {
           continue;
         }
 
-        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
+        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`, logger.tags.mining);
 
         for (const blockHeight of missingBlockHeights) {
           if (blockHeight < lastBlockToIndex) {
@@ -347,18 +483,18 @@ class Blocks {
           }
           ++indexedThisRun;
           ++totalIndexed;
-          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+          const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
           if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
             const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
             const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
             const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
             timer = new Date().getTime() / 1000;
             indexedThisRun = 0;
             loadingIndicators.setProgress('block-indexing', progress, false);
           }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
-          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+          const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
           const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
           const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -369,13 +505,13 @@ class Blocks {
         currentBlockHeight -= chunkSize;
       }
       if (newlyIndexed > 0) {
-        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       } else {
-        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       }
       loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
-      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       loadingIndicators.setProgress('block-indexing', 100);
       throw e;
     }
@@ -406,13 +542,13 @@ class Blocks {
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
-        const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+        const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
+          const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -427,6 +563,7 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        await chainTips.updateOrphanedBlocks();
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -437,18 +574,25 @@ class Blocks {
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
 
+      // start async callbacks
+      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
+
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
-          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
-            logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
+          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
+            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
             // We assume there won't be a reorg with more than 10 block depth
-            await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
+            await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
             for (let i = 10; i >= 0; --i) {
-              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
+              const newBlock = await this.$indexBlock(lastBlock.height - i);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
+              if (config.MEMPOOL.CPFP_INDEXING) {
+                await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+              }
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
@@ -457,10 +601,26 @@ class Blocks {
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
 
+          const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
+            await blocksRepository.$saveBlockPrices([{
+              height: blockExtended.height,
+              priceId: lastestPriceId,
+            }]);
+          } else {
+            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+            setTimeout(() => {
+              indexer.runSingleTask('blocksPrices');
+            }, 10000);
+          }
+
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
           }
+          if (config.MEMPOOL.CPFP_INDEXING) {
+            this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
+          }
         }
       }
 
@@ -491,9 +651,12 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority()) {
+      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
         diskCache.$saveCacheToDisk();
       }
+
+      // wait for pending async callbacks to finish
+      await Promise.all(callbackPromises);
     }
   }
 
@@ -501,23 +664,27 @@ class Blocks {
    * Index a block if it's missing from the database. Returns the block after indexing
    */
   public async $indexBlock(height: number): Promise<BlockExtended> {
-    const dbBlock = await blocksRepository.$getBlockByHeight(height);
-    if (dbBlock != null) {
-      return prepareBlock(dbBlock);
+    if (Common.indexingEnabled()) {
+      const dbBlock = await blocksRepository.$getBlockByHeight(height);
+      if (dbBlock !== null) {
+        return dbBlock;
+      }
     }
 
     const blockHash = await bitcoinApi.$getBlockHash(height);
-    const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
     const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
 
-    await blocksRepository.$saveBlockInDatabase(blockExtended);
+    if (Common.indexingEnabled()) {
+      await blocksRepository.$saveBlockInDatabase(blockExtended);
+    }
 
-    return prepareBlock(blockExtended);
+    return blockExtended;
   }
 
   /**
-   * Index a block by hash if it's missing from the database. Returns the block after indexing
+   * Get one block by its hash
    */
   public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
     // Check the memory cache
@@ -526,31 +693,14 @@ class Blocks {
       return blockByHash;
     }
 
-    // Block has already been indexed
-    if (Common.indexingEnabled()) {
-      const dbBlock = await blocksRepository.$getBlockByHash(hash);
-      if (dbBlock != null) {
-        return prepareBlock(dbBlock);
-      }
-    }
-
-    // Not Bitcoin network, return the block as it
+    // Not Bitcoin network, return the block as it from the bitcoin backend
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return await bitcoinApi.$getBlock(hash);
+      return await bitcoinCoreApi.$getBlock(hash);
     }
 
-    let block = await bitcoinClient.getBlock(hash);
-    block = prepareBlock(block);
-
     // Bitcoin network, add our custom data on top
-    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
-    const blockExtended = await this.$getBlockExtended(block, transactions);
-    if (Common.indexingEnabled()) {
-      delete(blockExtended['coinbaseTx']);
-      await blocksRepository.$saveBlockInDatabase(blockExtended);
-    }
-
-    return blockExtended;
+    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
+    return await this.$indexBlock(block.height);
   }
 
   public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@@ -559,7 +709,7 @@ class Blocks {
     if (skipMemoryCache === false) {
       // Check the memory cache
       const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);
-      if (cachedSummary) {
+      if (cachedSummary?.transactions?.length) {
         return cachedSummary.transactions;
       }
     }
@@ -567,7 +717,7 @@ class Blocks {
     // Check if it's indexed in db
     if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) {
       const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash);
-      if (indexedSummary !== undefined) {
+      if (indexedSummary !== undefined && indexedSummary?.transactions?.length) {
         return indexedSummary.transactions;
       }
     }
@@ -584,35 +734,39 @@ class Blocks {
     return summary.transactions;
   }
 
+  /**
+   * Get 15 blocks
+   * 
+   * Internally this function uses two methods to get the blocks, and
+   * the method is automatically selected:
+   *  - Using previous block hash links
+   *  - Using block height
+   * 
+   * @param fromHeight 
+   * @param limit 
+   * @returns 
+   */
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+    let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
+    if (currentHeight > this.currentBlockHeight) {
+      limit -= currentHeight - this.currentBlockHeight;
+      currentHeight = this.currentBlockHeight;
+    }
     const returnBlocks: BlockExtended[] = [];
 
     if (currentHeight < 0) {
       return returnBlocks;
     }
 
-    // Check if block height exist in local cache to skip the hash lookup
-    const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
-    let startFromHash: string | null = null;
-    if (blockByHeight) {
-      startFromHash = blockByHeight.id;
-    } else if (!Common.indexingEnabled()) {
-      startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
-    }
-
-    let nextHash = startFromHash;
     for (let i = 0; i < limit && currentHeight >= 0; i++) {
       let block = this.getBlocks().find((b) => b.height === currentHeight);
       if (block) {
+        // Using the memory cache (find by height)
         returnBlocks.push(block);
-      } else if (Common.indexingEnabled()) {
+      } else {
+        // Using indexing (find by height, index on the fly, save in database)
         block = await this.$indexBlock(currentHeight);
         returnBlocks.push(block);
-      } else if (nextHash != null) {
-        block = prepareBlock(await bitcoinClient.getBlock(nextHash));
-        nextHash = block.previousblockhash;
-        returnBlocks.push(block);
       }
       currentHeight--;
     }
@@ -620,6 +774,130 @@ class Blocks {
     return returnBlocks;
   }
 
+  /**
+   * Used for bulk block data query
+   * 
+   * @param fromHeight 
+   * @param toHeight 
+   */
+  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
+    if (!Common.indexingEnabled()) {
+      return [];
+    }
+
+    const blocks: any[] = [];
+
+    while (fromHeight <= toHeight) {
+      let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
+      if (!block) {
+        await this.$indexBlock(fromHeight);
+        block = await blocksRepository.$getBlockByHeight(fromHeight);
+        if (!block) {
+          continue;
+        }
+      }
+
+      // Cleanup fields before sending the response
+      const cleanBlock: any = {
+        height: block.height ?? null,
+        hash: block.id ?? null,
+        timestamp: block.timestamp ?? null,
+        median_timestamp: block.mediantime ?? null,
+        previous_block_hash: block.previousblockhash ?? null,
+        difficulty: block.difficulty ?? null,
+        header: block.extras.header ?? null,
+        version: block.version ?? null,
+        bits: block.bits ?? null,
+        nonce: block.nonce ?? null,
+        size: block.size ?? null,
+        weight: block.weight ?? null,
+        tx_count: block.tx_count ?? null,
+        merkle_root: block.merkle_root ?? null,
+        reward: block.extras.reward ?? null,
+        total_fee_amt: block.extras.totalFees ?? null,
+        avg_fee_amt: block.extras.avgFee ?? null,
+        median_fee_amt: block.extras.medianFeeAmt ?? null,
+        fee_amt_percentiles: block.extras.feePercentiles ?? null,
+        avg_fee_rate: block.extras.avgFeeRate ?? null,
+        median_fee_rate: block.extras.medianFee ?? null,
+        fee_rate_percentiles: block.extras.feeRange ?? null,
+        total_inputs: block.extras.totalInputs ?? null,
+        total_input_amt: block.extras.totalInputAmt ?? null,
+        total_outputs: block.extras.totalOutputs ?? null,
+        total_output_amt: block.extras.totalOutputAmt ?? null,
+        segwit_total_txs: block.extras.segwitTotalTxs ?? null,
+        segwit_total_size: block.extras.segwitTotalSize ?? null,
+        segwit_total_weight: block.extras.segwitTotalWeight ?? null,
+        avg_tx_size: block.extras.avgTxSize ?? null,
+        utxoset_change: block.extras.utxoSetChange ?? null,
+        utxoset_size: block.extras.utxoSetSize ?? null,
+        coinbase_raw: block.extras.coinbaseRaw ?? null,
+        coinbase_address: block.extras.coinbaseAddress ?? null,
+        coinbase_signature: block.extras.coinbaseSignature ?? null,
+        coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
+        pool_slug: block.extras.pool.slug ?? null,
+        pool_id: block.extras.pool.id ?? null,
+      };
+
+      if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
+        cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        if (cleanBlock.fee_amt_percentiles === null) {
+          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
+          const summary = this.summarizeBlock(block);
+          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        }
+        if (cleanBlock.fee_amt_percentiles !== null) {
+          cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+        }
+      }
+
+      cleanBlock.fee_amt_percentiles = {
+        'min': cleanBlock.fee_amt_percentiles[0],
+        'perc_10': cleanBlock.fee_amt_percentiles[1],
+        'perc_25': cleanBlock.fee_amt_percentiles[2],
+        'perc_50': cleanBlock.fee_amt_percentiles[3],
+        'perc_75': cleanBlock.fee_amt_percentiles[4],
+        'perc_90': cleanBlock.fee_amt_percentiles[5],
+        'max': cleanBlock.fee_amt_percentiles[6],
+      };
+      cleanBlock.fee_rate_percentiles = {
+        'min': cleanBlock.fee_rate_percentiles[0],
+        'perc_10': cleanBlock.fee_rate_percentiles[1],
+        'perc_25': cleanBlock.fee_rate_percentiles[2],
+        'perc_50': cleanBlock.fee_rate_percentiles[3],
+        'perc_75': cleanBlock.fee_rate_percentiles[4],
+        'perc_90': cleanBlock.fee_rate_percentiles[5],
+        'max': cleanBlock.fee_rate_percentiles[6],
+      };
+
+      // Re-org can happen after indexing so we need to always get the
+      // latest state from core
+      cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
+
+      blocks.push(cleanBlock);
+      fromHeight++;
+    }
+
+    return blocks;
+  }
+
+  public async $getBlockAuditSummary(hash: string): Promise<any> {
+    let summary;
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+      summary = await BlocksAuditsRepository.$getBlockAudit(hash);
+    }
+
+    // fallback to non-audited transaction summary
+    if (!summary?.transactions?.length) {
+      const strippedTransactions = await this.$getStrippedBlockTransactions(hash);
+      summary = {
+        transactions: strippedTransactions
+      };
+    }
+    return summary;
+  }
+
   public getLastDifficultyAdjustmentTime(): number {
     return this.lastDifficultyAdjustmentTime;
   }
@@ -631,6 +909,50 @@ class Blocks {
   public getCurrentBlockHeight(): number {
     return this.currentBlockHeight;
   }
+
+  public async $indexCPFP(hash: string, height: number): Promise<void> {
+    const block = await bitcoinClient.getBlock(hash, 2);
+    const transactions = block.tx.map(tx => {
+      tx.vsize = tx.weight / 4;
+      tx.fee *= 100_000_000;
+      return tx;
+    });
+
+    const clusters: any[] = [];
+
+    let cluster: TransactionStripped[] = [];
+    let ancestors: { [txid: string]: boolean } = {};
+    for (let i = transactions.length - 1; i >= 0; i--) {
+      const tx = transactions[i];
+      if (!ancestors[tx.txid]) {
+        let totalFee = 0;
+        let totalVSize = 0;
+        cluster.forEach(tx => {
+          totalFee += tx?.fee || 0;
+          totalVSize += tx.vsize;
+        });
+        const effectiveFeePerVsize = totalFee / totalVSize;
+        if (cluster.length > 1) {
+          clusters.push({
+            root: cluster[0].txid,
+            height,
+            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }),
+            effectiveFeePerVsize,
+          });
+        }
+        cluster = [];
+        ancestors = {};
+      }
+      cluster.push(tx);
+      tx.vin.forEach(vin => {
+        ancestors[vin.txid] = true;
+      });
+    }
+    const result = await cpfpRepository.$batchSaveClusters(clusters);
+    if (!result) {
+      await cpfpRepository.$insertProgressMarker(height);
+    }
+  }
 }
 
 export default new Blocks();

backend/src/api/chain-tips.ts

@@ -0,0 +1,61 @@
+import logger from '../logger';
+import bitcoinClient from './bitcoin/bitcoin-client';
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: OrphanedBlock[] = [];
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+      this.orphanedBlocks = [];
+
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
+          let block = await bitcoinClient.getBlock(chain.hash);
+          while (block && block.confirmations === -1) {
+            this.orphanedBlocks.push({
+              height: block.height,
+              hash: block.hash,
+              status: chain.status
+            });
+            block = await bitcoinClient.getBlock(block.previousblockhash);
+          }
+        }
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
+    if (height === undefined) {
+      return [];
+    }
+
+    const orphans: OrphanedBlock[] = [];
+    for (const block of this.orphanedBlocks) {
+      if (block.height === height) {
+        orphans.push(block);
+      }
+    }
+    return orphans;
+  }
+}
+
+export default new ChainTips();

backend/src/api/common.ts

@@ -1,5 +1,7 @@
-import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
+import { NodeSocket } from '../repositories/NodesSocketsRepository';
+import { isIP } from 'net';
 export class Common {
   static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
     '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -33,24 +35,31 @@ export class Common {
   }
 
   static getFeesInRange(transactions: TransactionExtended[], rangeLength: number) {
-    const arr = [transactions[transactions.length - 1].effectiveFeePerVsize];
+    const filtered: TransactionExtended[] = [];
+    let lastValidRate = Infinity;
+    // filter out anomalous fee rates to ensure monotonic range
+    for (const tx of transactions) {
+      if (tx.effectiveFeePerVsize <= lastValidRate) {
+        filtered.push(tx);
+        lastValidRate = tx.effectiveFeePerVsize;
+      }
+    }
+    const arr = [filtered[filtered.length - 1].effectiveFeePerVsize];
     const chunk = 1 / (rangeLength - 1);
     let itemsToAdd = rangeLength - 2;
 
     while (itemsToAdd > 0) {
-      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].effectiveFeePerVsize);
+      arr.push(filtered[Math.floor(filtered.length * chunk * itemsToAdd)].effectiveFeePerVsize);
       itemsToAdd--;
     }
 
-    arr.push(transactions[0].effectiveFeePerVsize);
+    arr.push(filtered[0].effectiveFeePerVsize);
     return arr;
   }
 
   static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } {
     const matches: { [txid: string]: TransactionExtended } = {};
     deleted
-      // The replaced tx must have at least one input with nSequence < maxint-1 (That’s the opt-in)
-      .filter((tx) => tx.vin.some((vin) => vin.sequence < 0xfffffffe))
       .forEach((deletedTx) => {
         const foundMatches = added.find((addedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
@@ -59,7 +68,7 @@ export class Common {
             && addedTx.feePerVsize > deletedTx.feePerVsize
             // Spends one or more of the same inputs
             && deletedTx.vin.some((deletedVin) =>
-              addedTx.vin.some((vin) => vin.txid === deletedVin.txid));
+              addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
         if (foundMatches) {
           matches[deletedTx.txid] = foundMatches;
@@ -155,6 +164,30 @@ export class Common {
     return parents;
   }
 
+  // calculates the ratio of matched transactions to projected transactions by weight
+  static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number {
+    let matchedWeight = 0;
+    let projectedWeight = 0;
+    const inBlock = {};
+
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const tx of projectedBlock.transactions) {
+      if (inBlock[tx.txid]) {
+        matchedWeight += tx.vsize * 4;
+      }
+      projectedWeight += tx.vsize * 4;
+    }
+
+    projectedWeight += transactions[0].weight;
+    matchedWeight += transactions[0].weight;
+
+    return projectedWeight ? matchedWeight / projectedWeight : 1;
+  }
+
   static getSqlInterval(interval: string | null): string | null {
     switch (interval) {
       case '24h': return '1 DAY';
@@ -166,6 +199,7 @@ export class Common {
       case '1y': return '1 YEAR';
       case '2y': return '2 YEAR';
       case '3y': return '3 YEAR';
+      case '4y': return '4 YEAR';
       default: return null;
     }
   }
@@ -184,4 +218,131 @@ export class Common {
       config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
     );
   }
+
+  static cpfpIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.CPFP_INDEXING === true
+    );
+  }
+
+  static setDateMidnight(date: Date): void {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+  }
+
+  static channelShortIdToIntegerId(channelId: string): string {
+    if (channelId.indexOf('x') === -1) { // Already an integer id
+      return channelId;
+    }
+    if (channelId.indexOf('/') !== -1) { // Topology import
+      channelId = channelId.slice(0, -2);
+    }
+    const s = channelId.split('x').map(part => BigInt(part));
+    return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
+  }
+
+  /** Decodes a channel id returned by lnd as uint64 to a short channel id */
+  static channelIntegerIdToShortId(id: string): string {
+    if (id.indexOf('/') !== -1) {
+      id = id.slice(0, -2);
+    }
+    
+    if (id.indexOf('x') !== -1) { // Already a short id
+      return id;
+    }
+
+    const n = BigInt(id);
+    return [
+      n >> 40n, // nth block
+      (n >> 16n) & 0xffffffn, // nth tx of the block
+      n & 0xffffn // nth output of the tx
+    ].join('x');
+  }
+
+  static utcDateToMysql(date?: number | null): string | null {
+    if (date === null) {
+      return null;
+    }
+    const d = new Date((date || 0) * 1000);
+    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
+  }
+
+  static findSocketNetwork(addr: string): {network: string | null, url: string} {
+    let network: string | null = null;
+    let url: string = addr;
+
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      url = addr.split('://')[1];
+    }
+
+    if (!url) {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    if (addr.indexOf('onion') !== -1) {
+      if (url.split('.')[0].length >= 56) {
+        network = 'torv3';
+      } else {
+        network = 'torv2';
+      }
+    } else if (addr.indexOf('i2p') !== -1) {
+      network = 'i2p';
+    } else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
+      const ipv = isIP(url.split(':')[0]);
+      if (ipv === 4) {
+        network = 'ipv4';
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
+      url = url.split('[')[1].split(']')[0];
+      const ipv = isIP(url);
+      if (ipv === 6) {
+        const parts = addr.split(':');
+        network = 'ipv6';
+        url = `[${url}]:${parts[parts.length - 1]}`;
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    return {
+      network: network,
+      url: url,
+    };
+  }
+
+  static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      return {
+        publicKey: publicKey,
+        network: socket.network,
+        addr: socket.addr,
+      };
+    } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
+      const formatted = this.findSocketNetwork(socket.addr);
+      return {
+        publicKey: publicKey,
+        network: formatted.network,
+        addr: formatted.url,
+      };
+    }
+  }
 }

backend/src/api/database-migration.ts

@@ -2,10 +2,13 @@ import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from './common';
+import blocksRepository from '../repositories/BlocksRepository';
+import cpfpRepository from '../repositories/CpfpRepository';
+import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 33;
-  private queryTimeout = 120000;
+  private static currentVersion = 59;
+  private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
 
@@ -59,8 +62,8 @@ class DatabaseMigration {
 
     if (databaseSchemaVersion <= 2) {
       // Disable some spam logs when they're not relevant
-      this.uniqueLogs.push(this.blocksTruncatedMessage);
-      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
     }
 
     logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -83,7 +86,7 @@ class DatabaseMigration {
       try {
         await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
         if (databaseSchemaVersion === 0) {
-          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
         } else {
           logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
         }
@@ -107,18 +110,22 @@ class DatabaseMigration {
     await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
     if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
       await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
+      await this.updateToSchemaVersion(2);
     }
     if (databaseSchemaVersion < 3) {
       await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+      await this.updateToSchemaVersion(3);
     }
     if (databaseSchemaVersion < 4) {
       await this.$executeQuery('DROP table IF EXISTS blocks;');
       await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
+      await this.updateToSchemaVersion(4);
     }
     if (databaseSchemaVersion < 5 && isBitcoin === true) {
       this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
       await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
       await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(5);
     }
 
     if (databaseSchemaVersion < 6 && isBitcoin === true) {
@@ -141,11 +148,13 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
       await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
+      await this.updateToSchemaVersion(6);
     }
 
     if (databaseSchemaVersion < 7 && isBitcoin === true) {
       await this.$executeQuery('DROP table IF EXISTS hashrates;');
       await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
+      await this.updateToSchemaVersion(7);
     }
 
     if (databaseSchemaVersion < 8 && isBitcoin === true) {
@@ -155,6 +164,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
       await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
+      await this.updateToSchemaVersion(8);
     }
 
     if (databaseSchemaVersion < 9 && isBitcoin === true) {
@@ -162,10 +172,12 @@ class DatabaseMigration {
       await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
       await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
       await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
+      await this.updateToSchemaVersion(9);
     }
 
     if (databaseSchemaVersion < 10 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
+      await this.updateToSchemaVersion(10);
     }
 
     if (databaseSchemaVersion < 11 && isBitcoin === true) {
@@ -178,11 +190,13 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(11);
     }
 
     if (databaseSchemaVersion < 12 && isBitcoin === true) {
       // No need to re-index because the new data type can contain larger values
       await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(12);
     }
 
     if (databaseSchemaVersion < 13 && isBitcoin === true) {
@@ -190,6 +204,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(13);
     }
 
     if (databaseSchemaVersion < 14 && isBitcoin === true) {
@@ -197,37 +212,45 @@ class DatabaseMigration {
       await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
       await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
       await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(14);
     }
 
     if (databaseSchemaVersion < 16 && isBitcoin === true) {
       this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
       await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
+      await this.updateToSchemaVersion(16);
     }
 
     if (databaseSchemaVersion < 17 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
+      await this.updateToSchemaVersion(17);
     }
 
     if (databaseSchemaVersion < 18 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
+      await this.updateToSchemaVersion(18);
     }
 
     if (databaseSchemaVersion < 19) {
       await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
+      await this.updateToSchemaVersion(19);
     }
 
     if (databaseSchemaVersion < 20 && isBitcoin === true) {
       await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
+      await this.updateToSchemaVersion(20);
     }
 
     if (databaseSchemaVersion < 21) {
       await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
       await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
+      await this.updateToSchemaVersion(21);
     }
 
     if (databaseSchemaVersion < 22 && isBitcoin === true) {
       await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
       await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
+      await this.updateToSchemaVersion(22);
     }
 
     if (databaseSchemaVersion < 23) {
@@ -240,19 +263,21 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
+      await this.updateToSchemaVersion(23);
     }
 
     if (databaseSchemaVersion < 24 && isBitcoin == true) {
       await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
       await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
+      await this.updateToSchemaVersion(24);
     }
 
     if (databaseSchemaVersion < 25 && isBitcoin === true) {
-      await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
       await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
       await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
       await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
       await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
+      await this.updateToSchemaVersion(25);
     }
 
     if (databaseSchemaVersion < 26 && isBitcoin === true) {
@@ -263,6 +288,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(26);
     }
 
     if (databaseSchemaVersion < 27 && isBitcoin === true) {
@@ -272,8 +298,9 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(27);
     }
-    
+
     if (databaseSchemaVersion < 28 && isBitcoin === true) {
       if (config.LIGHTNING.ENABLED) {
         this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
@@ -281,6 +308,7 @@ class DatabaseMigration {
       await this.$executeQuery(`TRUNCATE lightning_stats`);
       await this.$executeQuery(`TRUNCATE node_stats`);
       await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
+      await this.updateToSchemaVersion(28);
     }
 
     if (databaseSchemaVersion < 29 && isBitcoin === true) {
@@ -292,24 +320,200 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
+      await this.updateToSchemaVersion(29);
     }
 
     if (databaseSchemaVersion < 30 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
+      await this.updateToSchemaVersion(30);
     }
 
     if (databaseSchemaVersion < 31 && isBitcoin == true) { // Link blocks to prices
       await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
       await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
       await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
+      await this.updateToSchemaVersion(31);
     }
 
     if (databaseSchemaVersion < 32 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(32);
     }
 
     if (databaseSchemaVersion < 33 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
+      await this.updateToSchemaVersion(33);
+    }
+
+    if (databaseSchemaVersion < 34 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(34);
+    }
+
+    if (databaseSchemaVersion < 35 && isBitcoin == true) {
+      await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
+      await this.updateToSchemaVersion(35);
+    }
+
+    if (databaseSchemaVersion < 36 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
+      await this.updateToSchemaVersion(36);
+    }
+
+    if (databaseSchemaVersion < 37 && isBitcoin == true) {
+      await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+      await this.updateToSchemaVersion(37);
+    }
+
+    if (databaseSchemaVersion < 38 && isBitcoin == true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
+      }
+      await this.$executeQuery(`TRUNCATE lightning_stats`);
+      await this.$executeQuery(`TRUNCATE node_stats`);
+      await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
+      await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
+      await this.updateToSchemaVersion(38);
+    }
+
+    if (databaseSchemaVersion < 39 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
+      await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
+      await this.updateToSchemaVersion(39);
+    }
+
+    if (databaseSchemaVersion < 40 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
+      await this.updateToSchemaVersion(40);
+    }
+
+    if (databaseSchemaVersion < 41 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
+      await this.updateToSchemaVersion(41);
+    }
+
+    if (databaseSchemaVersion < 42 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
+      await this.updateToSchemaVersion(42);
+    }
+
+    if (databaseSchemaVersion < 43 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+      await this.updateToSchemaVersion(43);
+    }
+
+    if (databaseSchemaVersion < 44 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE blocks_summaries SET template = NULL');
+      await this.updateToSchemaVersion(44);
+    }
+
+    if (databaseSchemaVersion < 45 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(45);
+    }
+
+    if (databaseSchemaVersion < 46) {
+      await this.$executeQuery(`ALTER TABLE blocks MODIFY blockTimestamp timestamp NOT NULL DEFAULT 0`);
+      await this.updateToSchemaVersion(46);
+    }
+
+    if (databaseSchemaVersion < 47) {
+      await this.$executeQuery('ALTER TABLE `blocks` ADD cpfp_indexed tinyint(1) DEFAULT 0');
+      await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
+      await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
+      await this.updateToSchemaVersion(47);
+    }
+
+    if (databaseSchemaVersion < 48 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node1_funding_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node2_funding_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node1_closing_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node2_closing_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD funding_ratio float unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `channels` ADD closed_by varchar(66) DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `channels` ADD single_funded tinyint(1) DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD outputs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(48);
+    }
+
+    if (databaseSchemaVersion < 49 && isBitcoin === true) {
+      await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
+      await this.updateToSchemaVersion(49);
+    }
+
+    if (databaseSchemaVersion < 50) {
+      await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`');
+      await this.updateToSchemaVersion(50);
+    }
+
+    if (databaseSchemaVersion < 51) {
+      await this.$executeQuery('ALTER TABLE `cpfp_clusters` ADD INDEX `height` (`height`)');
+      await this.updateToSchemaVersion(51);
+    }
+
+    if (databaseSchemaVersion < 52) {
+      await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters'));
+      await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions'));
+      try {
+        await this.$convertCompactCpfpTables();
+        await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
+        await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
+        await this.updateToSchemaVersion(52);
+      } catch (e) {
+        logger.warn('' + (e instanceof Error ? e.message : e));
+      }
+    }
+
+    if (databaseSchemaVersion < 53) {
+      await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
+      await this.updateToSchemaVersion(53);
+    }
+
+    if (databaseSchemaVersion < 54) {
+      this.uniqueLog(logger.notice, `'prices' table has been truncated`);
+      await this.$executeQuery(`TRUNCATE prices`);
+      if (isBitcoin === true) {
+        this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
+        await this.$executeQuery(`TRUNCATE blocks_prices`);
+      }
+      await this.updateToSchemaVersion(54);
+    }
+
+    if (databaseSchemaVersion < 55) {
+      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+      await this.updateToSchemaVersion(55);
+    }
+
+    if (databaseSchemaVersion < 56) {
+      await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
+      await this.$executeQuery('TRUNCATE TABLE `blocks`');
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('DELETE FROM `pools`');
+      await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+      this.uniqueLog(logger.notice, '`pools` table has been truncated`');
+      await this.updateToSchemaVersion(56);
+    }
+
+    if (databaseSchemaVersion < 57 && isBitcoin === true) {
+      await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
+      await this.updateToSchemaVersion(57);
+    }
+
+    if (databaseSchemaVersion < 58) {
+      // We only run some migration queries for this version
+      await this.updateToSchemaVersion(58);
+    }
+
+    if (databaseSchemaVersion < 59 && (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet')) {
+      // https://github.com/mempool/mempool/issues/3360
+      await this.$executeQuery(`TRUNCATE prices`);
     }
   }
 
@@ -434,10 +638,15 @@ class DatabaseMigration {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
     }
 
-    if (version < 9  && isBitcoin === true) {
+    if (version < 9 && isBitcoin === true) {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
     }
 
+    if (version < 58) {
+      queries.push(`DELETE FROM state WHERE name = 'last_hashrates_indexing'`);
+      queries.push(`DELETE FROM state WHERE name = 'last_weekly_hashrates_indexing'`);
+    }
+
     return queries;
   }
 
@@ -448,6 +657,10 @@ class DatabaseMigration {
     return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
   }
 
+  private async updateToSchemaVersion(version): Promise<void> {
+    await this.$executeQuery(`UPDATE state SET number = ${version} WHERE name = 'schema_version';`);
+  }
+
   /**
    * Print current database version
    */
@@ -580,6 +793,28 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getAdditionalBlocksDataQuery(): string {
+    return `ALTER TABLE blocks
+      ADD median_timestamp timestamp NOT NULL,
+      ADD coinbase_address varchar(100) NULL,
+      ADD coinbase_signature varchar(500) NULL,
+      ADD coinbase_signature_ascii varchar(500) NULL,
+      ADD avg_tx_size double unsigned NOT NULL,
+      ADD total_inputs int unsigned NOT NULL,
+      ADD total_outputs int unsigned NOT NULL,
+      ADD total_output_amt bigint unsigned NOT NULL,
+      ADD fee_percentiles longtext NULL,
+      ADD median_fee_amt int unsigned NULL,
+      ADD segwit_total_txs int unsigned NOT NULL,
+      ADD segwit_total_size int unsigned NOT NULL,
+      ADD segwit_total_weight int unsigned NOT NULL,
+      ADD header varchar(160) NOT NULL,
+      ADD utxoset_change int NOT NULL,
+      ADD utxoset_size int unsigned NULL,
+      ADD total_input_amt bigint unsigned NULL
+    `;
+  }
+
   private getCreateDailyStatsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS hashrates (
       hashrate_timestamp timestamp NOT NULL,
@@ -724,7 +959,7 @@ class DatabaseMigration {
       names text DEFAULT NULL,
       UNIQUE KEY id (id,type),
       KEY id_2 (id)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
   private getCreateBlocksPricesTableQuery(): string {
@@ -736,24 +971,119 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates', 'prices'];
+  private getCreateLNNodesSocketsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_sockets (
+      public_key varchar(66) NOT NULL,
+      socket varchar(100) NOT NULL,
+      type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL,
+      UNIQUE KEY public_key_socket (public_key, socket),
+      INDEX (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
 
+  private getCreateLNNodeRecordsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_records (
+      public_key varchar(66) NOT NULL,
+      type int(10) unsigned NOT NULL,
+      payload blob NOT NULL,
+      UNIQUE KEY public_key_type (public_key, type),
+      INDEX (public_key),
+      FOREIGN KEY (public_key)
+        REFERENCES nodes (public_key)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCPFPTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS cpfp_clusters (
+      root varchar(65) NOT NULL,
+      height int(10) NOT NULL,
+      txs JSON DEFAULT NULL,
+      fee_rate double unsigned NOT NULL,
+      PRIMARY KEY (root)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateTransactionsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS transactions (
+      txid varchar(65) NOT NULL,
+      cluster varchar(65) DEFAULT NULL,
+      PRIMARY KEY (txid),
+      FOREIGN KEY (cluster) REFERENCES cpfp_clusters (root) ON DELETE SET NULL
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCompactCPFPTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS compact_cpfp_clusters (
+      root binary(32) NOT NULL,
+      height int(10) NOT NULL,
+      txs BLOB DEFAULT NULL,
+      fee_rate float unsigned,
+      PRIMARY KEY (root),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCompactTransactionsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS compact_transactions (
+      txid binary(32) NOT NULL,
+      cluster binary(32) DEFAULT NULL,
+      PRIMARY KEY (txid)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  public async $blocksReindexingTruncate(): Promise<void> {
+    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
+    await Common.sleep$(5000);
+
+    await this.$executeQuery(`TRUNCATE blocks`);
+    await this.$executeQuery(`TRUNCATE hashrates`);
+    await this.$executeQuery(`TRUNCATE difficulty_adjustments`);
+    await this.$executeQuery('DELETE FROM `pools`');
+    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+  }
+
+  private async $convertCompactCpfpTables(): Promise<void> {
     try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
+      const batchSize = 250;
+      const maxHeight = await blocksRepository.$mostRecentBlockHeight() || 0;
+      const [minHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`);
+      const minHeight = (minHeightRows.length && minHeightRows[0].minHeight != null) ? minHeightRows[0].minHeight : maxHeight;
+      let height = maxHeight;
+
+      // Logging
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+
+      while (height > minHeight) {
+        const [rows] = await DB.query(
+          `
+            SELECT * from cpfp_clusters
+            WHERE height <= ? AND height > ?
+            ORDER BY height
+          `,
+          [height, height - batchSize]
+        ) as RowDataPacket[][];
+        if (rows?.length) {
+          await cpfpRepository.$batchSaveClusters(rows.map(row => {
+            return {
+              root: row.root,
+              height: row.height,
+              txs: JSON.parse(row.txs),
+              effectiveFeePerVsize: row.fee_rate,
+            };
+          }));
         }
 
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
+        const elapsed = new Date().getTime() / 1000 - timer;
+        const runningFor = new Date().getTime() / 1000 - startedAt;
+        logger.debug(`Migrated cpfp data from block ${height} to ${height - batchSize} in ${elapsed.toFixed(2)} seconds | total elapsed: ${runningFor.toFixed(2)} seconds`);
+        timer = new Date().getTime() / 1000;
+        height -= batchSize;
       }
     } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
+      logger.warn(`Failed to migrate cpfp transaction data`);
     }
   }
 }

backend/src/api/difficulty-adjustment.ts

@@ -2,65 +2,104 @@ import config from '../config';
 import { IDifficultyAdjustment } from '../mempool.interfaces';
 import blocks from './blocks';
 
-class DifficultyAdjustmentApi {
-  constructor() { }
+export interface DifficultyAdjustment {
+  progressPercent: number;       // Percent: 0 to 100
+  difficultyChange: number;      // Percent: -75 to 300
+  estimatedRetargetDate: number; // Unix time in ms
+  remainingBlocks: number;       // Block count
+  remainingTime: number;         // Duration of time in ms
+  previousRetarget: number;      // Percent: -75 to 300
+  previousTime: number;          // Unix time in ms
+  nextRetargetHeight: number;    // Block Height
+  timeAvg: number;               // Duration of time in ms
+  timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
+  expectedBlocks: number;         // Block count
+}
 
-  public getDifficultyAdjustment(): IDifficultyAdjustment {
+export function calcDifficultyAdjustment(
+  DATime: number,
+  nowSeconds: number,
+  blockHeight: number,
+  previousRetarget: number,
+  network: string,
+  latestBlockTimestamp: number,
+): DifficultyAdjustment {
+  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
+  const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
+  const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
+  const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
+
+  const diffSeconds = nowSeconds - DATime;
+  const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
+  const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
+  const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
+  const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
+  const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
+
+  let difficultyChange = 0;
+  let timeAvgSecs = diffSeconds / blocksInEpoch;
+  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+    // Max increase is x4 (+300%)
+    if (difficultyChange > 300) {
+      difficultyChange = 300;
+    }
+    // Max decrease is /4 (-75%)
+    if (difficultyChange < -75) {
+      difficultyChange = -75;
+    }
+  }
+
+  // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+  // therefore the time between blocks will always be below 20 minutes (1200s).
+  let timeOffset = 0;
+  if (network === 'testnet') {
+    if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
+    }
+
+    const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
+    if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
+    }
+  }
+
+  const timeAvg = Math.floor(timeAvgSecs * 1000);
+  const remainingTime = remainingBlocks * timeAvg;
+  const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
+
+  return {
+    progressPercent,
+    difficultyChange,
+    estimatedRetargetDate,
+    remainingBlocks,
+    remainingTime,
+    previousRetarget,
+    previousTime: DATime,
+    nextRetargetHeight,
+    timeAvg,
+    timeOffset,
+    expectedBlocks,
+  };
+}
+
+class DifficultyAdjustmentApi {
+  public getDifficultyAdjustment(): IDifficultyAdjustment | null {
     const DATime = blocks.getLastDifficultyAdjustmentTime();
     const previousRetarget = blocks.getPreviousDifficultyRetarget();
     const blockHeight = blocks.getCurrentBlockHeight();
     const blocksCache = blocks.getBlocks();
     const latestBlock = blocksCache[blocksCache.length - 1];
-
-    const now = new Date().getTime() / 1000;
-    const diff = now - DATime;
-    const blocksInEpoch = blockHeight % 2016;
-    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
-    const remainingBlocks = 2016 - blocksInEpoch;
-    const nextRetargetHeight = blockHeight + remainingBlocks;
-
-    let difficultyChange = 0;
-    if (remainingBlocks < 1870) {
-      if (blocksInEpoch > 0) {
-        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
-      }
-      if (difficultyChange > 300) {
-        difficultyChange = 300;
-      }
-      if (difficultyChange < -75) {
-        difficultyChange = -75;
-      }
+    if (!latestBlock) {
+      return null;
     }
+    const nowSeconds = Math.floor(new Date().getTime() / 1000);
 
-    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
-
-    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
-    // therefore the time between blocks will always be below 20 minutes (1200s).
-    let timeOffset = 0;
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      if (timeAvgMins > 20) {
-        timeAvgMins = 20;
-      }
-      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
-        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
-      }
-    }
-
-    const timeAvg = timeAvgMins * 60 * 1000 ;
-    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
-    const estimatedRetargetDate = remainingTime + now;
-
-    return {
-      progressPercent,
-      difficultyChange,
-      estimatedRetargetDate,
-      remainingBlocks,
-      remainingTime,
-      previousRetarget,
-      nextRetargetHeight,
-      timeAvg,
-      timeOffset,
-    };
+    return calcDifficultyAdjustment(
+      DATime, nowSeconds, blockHeight, previousRetarget,
+      config.MEMPOOL.NETWORK, latestBlock.timestamp
+    );
   }
 }
 

backend/src/api/disk-cache.ts

@@ -9,25 +9,35 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
-  private cacheSchemaVersion = 1;
+  private cacheSchemaVersion = 3;
 
+  private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-cache.json';
+  private static TMP_FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/tmp-cache{number}.json';
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
   private static CHUNK_FILES = 25;
   private isWritingCache = false;
 
-  constructor() { }
+  constructor() {
+    if (!cluster.isPrimary) {
+      return;
+    }
+    process.on('SIGINT', (e) => {
+      this.$saveCacheToDisk(true);
+      process.exit(0);
+    });
+  }
 
-  async $saveCacheToDisk(): Promise<void> {
+  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
     if (!cluster.isPrimary) {
       return;
     }
     if (this.isWritingCache) {
-      logger.debug('Saving cache already in progress. Skipping.')
+      logger.debug('Saving cache already in progress. Skipping.');
       return;
     }
     try {
-      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
       this.isWritingCache = true;
 
       const mempool = memPool.getMempool();
@@ -40,19 +50,48 @@ class DiskCache {
 
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
-      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
-        cacheSchemaVersion: this.cacheSchemaVersion,
-        blocks: blocks.getBlocks(),
-        blockSummaries: blocks.getBlockSummaries(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+      if (sync) {
+        fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
         }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
+      } else {
+        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
+        }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
       }
+
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
     } catch (e) {
@@ -61,14 +100,29 @@ class DiskCache {
     }
   }
 
-  wipeCache() {
-    fs.unlinkSync(DiskCache.FILE_NAME);
+  wipeCache(): void {
+    logger.notice(`Wiping nodejs backend cache/cache*.json files`);
+    try {
+      fs.unlinkSync(DiskCache.FILE_NAME);
+    } catch (e: any) {
+      if (e?.code !== 'ENOENT') {
+        logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
+      }
+    }
+
     for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-      fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+      const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
+      try {
+        fs.unlinkSync(filename);
+      } catch (e: any) {
+        if (e?.code !== 'ENOENT') {
+          logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
+        }
+      }
     }
   }
 
-  loadMempoolCache() {
+  loadMempoolCache(): void {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
@@ -82,6 +136,10 @@ class DiskCache {
           logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
           return this.wipeCache();
         }
+        if (data.network && data.network !== config.MEMPOOL.NETWORK) {
+          logger.notice('Disk cache contains data from a different network. Clearing it and skipping the cache loading.');
+          return this.wipeCache();
+        }
 
         if (data.mempoolArray) {
           for (const tx of data.mempoolArray) {

backend/src/api/explorer/channels.api.ts

@@ -1,5 +1,9 @@
 import logger from '../../logger';
 import DB from '../../database';
+import nodesApi from './nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { Common } from '../common';
 
 class ChannelsApi {
   public async $getAllChannels(): Promise<any[]> {
@@ -13,32 +17,61 @@ class ChannelsApi {
     }
   }
 
-  public async $getAllChannelsGeo(publicKey?: string): Promise<any[]> {
+  public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
     try {
+      let select: string;
+      if (style === 'widget') {
+        select = `
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      } else {
+        select = `
+          nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      }
+
       const params: string[] = [];
-      let query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
-        nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
-        nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
-        nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude,
-        channels.capacity
-      FROM channels
-      JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
-      JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
-      WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
-        AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
+      let query = `SELECT ${select}
+        FROM channels
+        JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
+        JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
+        WHERE channels.status = 1
+          AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
+          AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
       `;
 
       if (publicKey !== undefined) {
         query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
         params.push(publicKey);
         params.push(publicKey);
+      } else {
+        query += ` AND channels.capacity > 1000000
+          GROUP BY nodes_1.public_key, nodes_2.public_key
+          ORDER BY channels.capacity DESC
+          LIMIT 10000
+        `;        
       }
 
       const [rows]: any = await DB.query(query, params);
-      return rows.map((row) => [
-        row.node1_public_key, row.node1_alias, row.node1_longitude, row.node1_latitude,
-        row.node2_public_key, row.node2_alias, row.node2_longitude, row.node2_latitude,
-        row.capacity]);
+      return rows.map((row) => {
+        if (style === 'widget') {
+          return [
+            row.node1_longitude, row.node1_latitude,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        } else {
+          return [
+            row.node1_public_key, row.node1_alias,
+            row.node1_longitude, row.node1_latitude,
+            row.node2_public_key, row.node2_alias,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        }
+      });
     } catch (e) {
       logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
       throw e;
@@ -48,7 +81,7 @@ class ChannelsApi {
   public async $searchChannelsById(search: string): Promise<any[]> {
     try {
       const searchStripped = search.replace('%', '') + '%';
-      const query = `SELECT id, short_id, capacity FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
+      const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
       const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
       return rows;
     } catch (e) {
@@ -57,9 +90,14 @@ class ChannelsApi {
     }
   }
 
-  public async $getChannelsByStatus(status: number): Promise<any[]> {
+  public async $getChannelsByStatus(status: number | number[]): Promise<any[]> {
     try {
-      const query = `SELECT * FROM channels WHERE status = ?`;
+      let query: string;
+      if (Array.isArray(status)) {
+        query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`;
+      } else {
+        query = `SELECT * FROM channels WHERE status = ?`;
+      }
       const [rows]: any = await DB.query(query, [status]);
       return rows;
     } catch (e) {
@@ -79,6 +117,32 @@ class ChannelsApi {
     }
   }
 
+  public async $getUnresolvedClosedChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
+    try {
+      const query = `
+        SELECT channels.*
+        FROM channels
+        WHERE channels.source_checked != 1
+      `;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
     try {
       const query = `SELECT * FROM channels WHERE created IS NULL`;
@@ -92,7 +156,31 @@ class ChannelsApi {
 
   public async $getChannel(id: string): Promise<any> {
     try {
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`;
+      const query = `
+        SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
+          n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
+          channels.*,
+          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
+        LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
+        WHERE (
+          ns1.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node1_public_key
+          )
+          AND ns2.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node2_public_key
+          )
+        )
+        AND channels.id = ?
+      `;
+
       const [rows]: any = await DB.query(query, [id]);
       if (rows[0]) {
         return this.convertChannel(rows[0]);
@@ -168,9 +256,14 @@ class ChannelsApi {
 
   public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
     try {
-      transactionIds = transactionIds.map((id) => '\'' + id + '\'');
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.transaction_id IN (${transactionIds.join(', ')}) OR channels.closing_transaction_id IN (${transactionIds.join(', ')})`;
-      const [rows]: any = await DB.query(query);
+      const query = `
+        SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
+      `;
+      const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
       const channels = rows.map((row) => this.convertChannel(row));
       return channels;
     } catch (e) {
@@ -179,17 +272,199 @@ class ChannelsApi {
     }
   }
 
+  public async $getChannelByClosingId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.closing_transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        rows[0].outputs = JSON.parse(rows[0].outputs);
+        return rows[0];
+      }
+    } catch (e) {
+      logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsByOpeningId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        return rows.map(row => {
+          row.outputs = JSON.parse(row.outputs);
+          return row;
+        });
+      }
+    } catch (e) {
+      logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_closing_balance = ?,
+          node2_closing_balance = ?,
+          closed_by = ?,
+          closing_fee = ?,
+          outputs = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_closing_balance || 0,
+        channelInfo.node2_closing_balance || 0,
+        channelInfo.closed_by,
+        channelInfo.closing_fee || 0,
+        JSON.stringify(channelInfo.outputs),
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_funding_balance = ?,
+          node2_funding_balance = ?,
+          funding_ratio = ?,
+          single_funded = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_funding_balance || 0,
+        channelInfo.node2_funding_balance || 0,
+        channelInfo.funding_ratio,
+        channelInfo.single_funded ? 1 : 0,
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $markChannelSourceChecked(id: string): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels
+        SET source_checked = 1
+        WHERE id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [id]);
+    } catch (e) {
+      logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
   public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
     try {
-      // Default active and inactive channels
-      let statusQuery = '< 2';
-      // Closed channels only
-      if (status === 'closed') {
-        statusQuery = '= 2';
+      let channelStatusFilter;
+      if (status === 'open') {
+        channelStatusFilter = '< 2';
+      } else if (status === 'active') {
+        channelStatusFilter = '= 1';
+      } else if (status === 'closed') {
+        channelStatusFilter = '= 2';
+      } else {
+        throw new Error('getChannelsForNode: Invalid status requested');
       }
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`;
-      const [rows]: any = await DB.query(query, [public_key, public_key, index, length]);
-      const channels = rows.map((row) => this.convertChannel(row));
+
+      // Channels originating from node
+      let query = `
+        SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
+          channels.status, channels.node1_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
+        WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsFromNode]: any = await DB.query(query, [public_key]);
+
+      // Channels incoming to node
+      query = `
+        SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
+          channels.status, channels.node2_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
+        WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsToNode]: any = await DB.query(query, [public_key]);
+
+      let allChannels = channelsFromNode.concat(channelsToNode);
+      allChannels.sort((a, b) => {
+        if (status === 'closed') {
+          if (!b.closing_date && !a.closing_date) {
+            return (b.updated_at ?? 0) - (a.updated_at ?? 0);
+          } else {
+            return (b.closing_date ?? 0) - (a.closing_date ?? 0);
+          }
+        } else {
+          return b.capacity - a.capacity;
+        }
+      });
+
+      if (index >= 0) {
+        allChannels = allChannels.slice(index, index + length);
+      } else if (index === -1) { // Node channels tree chart
+        allChannels = allChannels.slice(0, 1000);
+      }
+
+      const channels: any[] = []
+      for (const row of allChannels) {
+        let channel;
+        if (index >= 0) {
+          const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
+          channel = {
+            status: row.status,
+            closing_reason: row.closing_reason,
+            closing_date: row.closing_date,
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+              channels: activeChannelsStats.active_channel_count ?? 0,
+              capacity: activeChannelsStats.capacity ?? 0,
+            }
+          };
+        } else if (index === -1) {
+          channel = {
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+            }
+          };
+        }
+
+        channels.push(channel);
+      }
+
       return channels;
     } catch (e) {
       logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
@@ -205,7 +480,12 @@ class ChannelsApi {
       if (status === 'closed') {
         statusQuery = '= 2';
       }
-      const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`;
+      const query = `
+        SELECT COUNT(*) AS count
+        FROM channels
+        WHERE (node1_public_key = ? OR node2_public_key = ?)
+        AND status ${statusQuery}
+      `;
       const [rows]: any = await DB.query(query, [public_key, public_key]);
       return rows[0]['count'];
     } catch (e) {
@@ -222,10 +502,15 @@ class ChannelsApi {
       'transaction_id': channel.transaction_id,
       'transaction_vout': channel.transaction_vout,
       'closing_transaction_id': channel.closing_transaction_id,
+      'closing_fee': channel.closing_fee,
       'closing_reason': channel.closing_reason,
+      'closing_date': channel.closing_date,
       'updated_at': channel.updated_at,
       'created': channel.created,
       'status': channel.status,
+      'funding_ratio': channel.funding_ratio,
+      'closed_by': channel.closed_by,
+      'single_funded': !!channel.single_funded,
       'node_left': {
         'alias': channel.alias_left,
         'public_key': channel.node1_public_key,
@@ -238,6 +523,11 @@ class ChannelsApi {
         'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
         'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
         'updated_at': channel.node1_updated_at,
+        'longitude': channel.node1_longitude,
+        'latitude': channel.node1_latitude,
+        'funding_balance': channel.node1_funding_balance,
+        'closing_balance': channel.node1_closing_balance,
+        'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
       },
       'node_right': {
         'alias': channel.alias_right,
@@ -251,9 +541,169 @@ class ChannelsApi {
         'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
         'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
         'updated_at': channel.node2_updated_at,
+        'longitude': channel.node2_longitude,
+        'latitude': channel.node2_latitude,
+        'funding_balance': channel.node2_funding_balance,
+        'closing_balance': channel.node2_closing_balance,
+        'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
       },
     };
   }
+
+  /**
+   * Save or update a channel present in the graph
+   */
+  public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
+    const [ txid, vout ] = channel.chan_point.split(':');
+
+    const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
+    const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
+
+    // https://github.com/mempool/mempool/issues/3006
+    if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
+      channel.last_update = null;
+    }
+    if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy1.last_update = null;
+    }
+    if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy2.last_update = null;
+    }
+
+    const query = `INSERT INTO channels
+      (
+        id,
+        short_id,
+        capacity,
+        transaction_id,
+        transaction_vout,
+        updated_at,
+        status,
+        node1_public_key,
+        node1_base_fee_mtokens,
+        node1_cltv_delta,
+        node1_fee_rate,
+        node1_is_disabled,
+        node1_max_htlc_mtokens,
+        node1_min_htlc_mtokens,
+        node1_updated_at,
+        node2_public_key,
+        node2_base_fee_mtokens,
+        node2_cltv_delta,
+        node2_fee_rate,
+        node2_is_disabled,
+        node2_max_htlc_mtokens,
+        node2_min_htlc_mtokens,
+        node2_updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+        capacity = ?,
+        updated_at = ?,
+        status = ${status},
+        node1_public_key = ?,
+        node1_base_fee_mtokens = ?,
+        node1_cltv_delta = ?,
+        node1_fee_rate = ?,
+        node1_is_disabled = ?,
+        node1_max_htlc_mtokens = ?,
+        node1_min_htlc_mtokens = ?,
+        node1_updated_at = ?,
+        node2_public_key = ?,
+        node2_base_fee_mtokens = ?,
+        node2_cltv_delta = ?,
+        node2_fee_rate = ?,
+        node2_is_disabled = ?,
+        node2_max_htlc_mtokens = ?,
+        node2_min_htlc_mtokens = ?,
+        node2_updated_at = ?
+      ;`;
+
+    await DB.query(query, [
+      Common.channelShortIdToIntegerId(channel.channel_id),
+      Common.channelIntegerIdToShortId(channel.channel_id),
+      channel.capacity,
+      txid,
+      vout,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update),
+      channel.capacity,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update)
+    ]);
+  }
+
+  /**
+   * Set all channels not in `graphChannelsIds` as inactive (status = 0)
+   */
+  public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
+    if (graphChannelsIds.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE id NOT IN (
+          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
+        )
+        AND status != 2
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
+    try {
+      const query = `
+        SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
+        FROM channels
+        WHERE node1_public_key = ?
+      `;
+      const [rows]: any[] = await DB.query(query, [publicKey]);
+      if (rows.length > 0) {
+        return rows[0].updated_at;
+      }
+    } catch (e) {
+      logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
+    }
+    return 0;
+  }
 }
 
 export default new ChannelsApi();

backend/src/api/explorer/channels.routes.ts

@@ -32,6 +32,9 @@ class ChannelsRoutes {
         res.status(404).send('Channel not found');
         return;
       }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channel);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -44,11 +47,22 @@ class ChannelsRoutes {
         res.status(400).send('Missing parameter: public_key');
         return;
       }
+
       const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
       const status: string = typeof req.query.status === 'string' ? req.query.status : '';
-      const length = 25;
-      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status);
+
+      if (index < -1) {
+        res.status(400).send('Invalid index');
+      }
+      if (['open', 'active', 'closed'].includes(status) === false) {
+        res.status(400).send('Invalid status');
+      }
+
+      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
       const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.header('X-Total-Count', channelsCount.toString());
       res.json(channels);
     } catch (e) {
@@ -56,7 +70,7 @@ class ChannelsRoutes {
     }
   }
 
-  private async $getChannelsByTransactionIds(req: Request, res: Response) {
+  private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
     try {
       if (!Array.isArray(req.query.txId)) {
         res.status(400).send('Not an array');
@@ -69,27 +83,26 @@ class ChannelsRoutes {
         }
       }
       const channels = await channelsApi.$getChannelsByTransactionId(txIds);
-      const inputs: any[] = [];
-      const outputs: any[] = [];
+      const result: any[] = [];
       for (const txid of txIds) {
-        const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid);
-        if (foundChannelInputs) {
-          inputs.push(foundChannelInputs);
-        } else {
-          inputs.push(null);
+        const inputs: any = {};
+        const outputs: any = {};
+        // Assuming that we only have one lightning close input in each transaction. This may not be true in the future
+        const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
+        if (foundChannelsFromInput) {
+          inputs[0] = foundChannelsFromInput;
         }
-        const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid);
-        if (foundChannelOutputs) {
-          outputs.push(foundChannelOutputs);
-        } else {
-          outputs.push(null);
+        const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
+        for (const output of foundChannelsFromOutputs) {
+          outputs[output.transaction_vout] = output;
         }
+        result.push({
+          inputs,
+          outputs,
+        });
       }
 
-      res.json({
-        inputs: inputs,
-        outputs: outputs,
-      });
+      res.json(result);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
@@ -97,7 +110,11 @@ class ChannelsRoutes {
 
   private async $getAllChannelsGeo(req: Request, res: Response) {
     try {
-      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey);
+      const style: string = typeof req.query.style === 'string' ? req.query.style : '';
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channels);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);

backend/src/api/explorer/nodes.api.ts

@@ -1,24 +1,62 @@
 import logger from '../../logger';
 import DB from '../../database';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
 
 class NodesApi {
+  public async $getWorldNodes(): Promise<any> {
+    try {
+      let query = `
+        SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+        CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+        CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+        nodes.longitude, nodes.latitude,
+        geo_names_country.names as country, geo_names_iso.names as isoCode
+        FROM nodes
+        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+        ORDER BY capacity
+      `;
+
+      const [nodes]: any[] = await DB.query(query);
+
+      for (let i = 0; i < nodes.length; ++i) {
+        nodes[i].country = JSON.parse(nodes[i].country);
+      }
+
+      query = `
+        SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
+        FROM nodes
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+      `;
+
+      const [maximums]: any[] = await DB.query(query);
+      
+      return {
+        maxLiquidity: maximums[0].maxLiquidity,
+        maxChannels: maximums[0].maxChannels,
+        nodes: nodes.map(node => [
+          node.longitude, node.latitude,
+          node.publicKey, node.alias, node.capacity, node.channels,
+          node.country, node.isoCode
+        ])
+      };
+    } catch (e) {
+      logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
   public async $getNode(public_key: string): Promise<any> {
     try {
-      const query = `
-        SELECT nodes.*, geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
-        geo_names_country.names as country, geo_names_subdivision.names as subdivision,
-          (SELECT Count(*)
-          FROM channels
-          WHERE channels.status = 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_closed_count,
-          (SELECT Count(*)
-          FROM channels
-          WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_active_count,
-          (SELECT Sum(capacity)
-          FROM channels
-          WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity,
-          (SELECT Avg(capacity)
-          FROM channels
-          WHERE status = 1 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg
+      // General info
+      let query = `
+        SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
+          UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
+          as_number, city_id, country_id, subdivision_id, longitude, latitude,
+          geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
+          geo_names_country.names as country, geo_names_subdivision.names as subdivision
         FROM nodes
         LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
         LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
@@ -27,17 +65,128 @@ class NodesApi {
         LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
         WHERE public_key = ?
       `;
-      const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key]);
-      if (rows.length > 0) {
-        rows[0].as_organization = JSON.parse(rows[0].as_organization);
-        rows[0].subdivision = JSON.parse(rows[0].subdivision);
-        rows[0].city = JSON.parse(rows[0].city);
-        rows[0].country = JSON.parse(rows[0].country);
-        return rows[0];
+      let [rows]: any[] = await DB.query(query, [public_key]);
+      if (rows.length === 0) {
+        throw new Error(`This node does not exist, or our node is not seeing it yet`);
       }
-      return null;
+
+      const node = rows[0];
+      node.as_organization = JSON.parse(node.as_organization);
+      node.subdivision = JSON.parse(node.subdivision);
+      node.city = JSON.parse(node.city);
+      node.country = JSON.parse(node.country);
+
+      // Active channels and capacity
+      const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
+      node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
+      node.capacity = activeChannelsStats.capacity ?? 0;
+
+      // Opened channels count
+      query = `
+        SELECT count(short_id) as opened_channel_count
+        FROM channels
+        WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.opened_channel_count = 0;
+      if (rows.length > 0) {
+        node.opened_channel_count = rows[0].opened_channel_count;
+      }
+
+      // Closed channels count
+      query = `
+        SELECT count(short_id) as closed_channel_count
+        FROM channels
+        WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.closed_channel_count = 0;
+      if (rows.length > 0) {
+        node.closed_channel_count = rows[0].closed_channel_count;
+      }
+
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
+      return node;
     } catch (e) {
-      logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
+    const query = `
+      SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
+      FROM channels
+      WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+    `;
+    const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
+    if (rows.length > 0) {
+      return {
+        active_channel_count: rows[0].active_channel_count,
+        capacity: rows[0].capacity
+      };
+    } else {
+      return null;
+    }
+  }
+
+  public async $getFeeHistogram(node_public_key: string): Promise<unknown> {
+    try {
+      const inQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node2_fee_rate WHEN node2_public_key = ? THEN node1_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [inRows]: any[] = await DB.query(inQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      const outQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node1_fee_rate WHEN node2_public_key = ? THEN node2_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [outRows]: any[] = await DB.query(outQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      return {
+        incoming: inRows.length > 0 ? inRows : [],
+        outgoing: outRows.length > 0 ? outRows : [],
+      };
+    } catch (e) {
+      logger.err(`Cannot get node fee distribution for ${node_public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -55,7 +204,12 @@ class NodesApi {
 
   public async $getNodeStats(public_key: string): Promise<any> {
     try {
-      const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;
+      const query = `
+        SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
+        FROM node_stats
+        WHERE public_key = ?
+        ORDER BY added DESC
+      `;
       const [rows]: any = await DB.query(query, [public_key]);
       return rows;
     } catch (e) {
@@ -64,10 +218,44 @@ class NodesApi {
     }
   }
 
-  public async $getTopCapacityNodes(): Promise<any> {
+  public async $getTopCapacityNodes(full: boolean): Promise<ITopNodesPerCapacity[]> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query);
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.capacity
+          FROM nodes
+          ORDER BY capacity DESC
+          LIMIT 6
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
       return rows;
     } catch (e) {
       logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
@@ -75,10 +263,107 @@ class NodesApi {
     }
   }
 
-  public async $getTopChannelsNodes(): Promise<any> {
+  public async $getTopChannelsNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query);
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT
+            nodes.public_key as publicKey,
+            IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 6;
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
+      const latestDate = rows[0].maxAdded;
+
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            node_stats.channels
+          FROM node_stats
+          JOIN nodes ON nodes.public_key = node_stats.public_key
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
+            CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM node_stats
+          RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
       return rows;
     } catch (e) {
       logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
@@ -88,9 +373,16 @@ class NodesApi {
 
   public async $searchNodeByPublicKeyOrAlias(search: string) {
     try {
-      const searchStripped = search.replace('%', '') + '%';
-      const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR nodes.alias LIKE ? GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
+      const publicKeySearch = search.replace('%', '') + '%';
+      const aliasSearch = search
+        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".  
+        .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
+        .split(' ')
+        .filter(key => key.length)
+        .map((search) => '+' + search + '*').join(' ');
+      // %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
+      const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
       return rows;
     } catch (e) {
       logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
@@ -98,64 +390,120 @@ class NodesApi {
     }
   }
 
-  public async $getNodesISP(groupBy: string, showTor: boolean) {
+  public async $getNodesISPRanking() {
     try {
-      const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`;
-      
-      // Clearnet
-      let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names,
-          COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
-        FROM nodes
-        JOIN geo_names ON geo_names.id = nodes.as_number
-        JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
-        GROUP BY geo_names.names
-        ORDER BY ${orderBy} DESC
-      `;      
-      const [nodesCountPerAS]: any = await DB.query(query);
+      let query = '';
 
-      let total = 0;
-      const nodesPerAs: any[] = [];
+      // List all channels and the two linked ISP
+      query = `
+        SELECT short_id, channels.capacity,
+          channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1
+        ORDER BY short_id DESC
+      `;
+      const [channelsIsp]: any = await DB.query(query);
 
-      for (const asGroup of nodesCountPerAS) {
-        if (groupBy === 'capacity') {
-          total += asGroup.capacity;
-        } else {
-          total += asGroup.nodesCount;
+      // Sum channels capacity and node count per ISP
+      const ispList = {};
+      for (const channel of channelsIsp) {
+        const isp1 = JSON.parse(channel.isp1);
+        const isp2 = JSON.parse(channel.isp2);
+
+        if (!ispList[isp1]) {
+          ispList[isp1] = {
+            ids: [channel.isp1ID],
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp1].ids.includes(channel.isp1ID) === false) {
+          ispList[isp1].ids.push(channel.isp1ID);
         }
+
+        if (!ispList[isp2]) {
+          ispList[isp2] = {
+            ids: [channel.isp2ID],
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp2].ids.includes(channel.isp2ID) === false) {
+          ispList[isp2].ids.push(channel.isp2ID);
+        }
+        
+        ispList[isp1].capacity += channel.capacity;
+        ispList[isp1].channels += 1;
+        ispList[isp1].nodes[channel.node1PublicKey] = true;
+        ispList[isp2].capacity += channel.capacity;
+        ispList[isp2].channels += 1;
+        ispList[isp2].nodes[channel.node2PublicKey] = true;
+      }
+      
+      const ispRanking: any[] = [];
+      for (const isp of Object.keys(ispList)) {
+        ispRanking.push([
+          ispList[isp].ids.sort((a, b) => a - b).join(','),
+          isp,
+          ispList[isp].capacity,
+          ispList[isp].channels,
+          Object.keys(ispList[isp].nodes).length,
+        ]);
       }
 
-      // Tor
-      if (showTor) {
-        query = `SELECT COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
-          FROM nodes
-          JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
-          ORDER BY ${orderBy} DESC
-        `;      
-        const [nodesCountTor]: any = await DB.query(query);
+      // Total active channels capacity
+      query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`;
+      const [totalCapacity]: any = await DB.query(query);
 
-        total += groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount;
-        nodesPerAs.push({
-          ispId: null,
-          name: 'Tor',
-          count: nodesCountTor[0].nodesCount,
-          share: Math.floor((groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount) / total * 10000) / 100,
-          capacity: nodesCountTor[0].capacity,
-        });
-      }
+      // Get the total capacity of all channels which have at least one node on clearnet
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks LIKE '%ipv%'
+      `;
+      const [clearnetCapacity]: any = await DB.query(query);
 
-      for (const as of nodesCountPerAS) {
-        nodesPerAs.push({
-          ispId: as.ispId,
-          name: JSON.parse(as.names),
-          count: as.nodesCount,
-          share: Math.floor((groupBy === 'capacity' ? as.capacity : as.nodesCount) / total * 10000) / 100,
-          capacity: as.capacity,
-        });
-      }
+      // Get the total capacity of all channels which have both nodes on Tor 
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks NOT LIKE '%ipv%' AND
+          channels_tmp.networks NOT LIKE '%dns%' AND
+          channels_tmp.networks NOT LIKE '%websocket%'
+      `;
+      const [torCapacity]: any = await DB.query(query);
 
-      return nodesPerAs;
+      const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10);
+      const torCapacityValue = parseInt(torCapacity[0].capacity, 10);
+      const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue;
+
+      return {
+        clearnetCapacity: clearnetCapacityValue,
+        torCapacity: torCapacityValue,
+        unknownCapacity: unknownCapacityValue,
+        ispRanking: ispRanking,
+      };
     } catch (e) {
-      logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
     }
   }
@@ -163,25 +511,27 @@ class NodesApi {
   public async $getNodesPerCountry(countryId: string) {
     try {
       const query = `
-        SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
-          UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
-          geo_names_city.names as city
-        FROM node_stats
-        JOIN (
-          SELECT public_key, MAX(added) as last_added
-          FROM node_stats
-          GROUP BY public_key
-        ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
-        JOIN nodes ON nodes.public_key = node_stats.public_key
-        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
         WHERE geo_names_country.id = ?
         ORDER BY capacity DESC
       `;
 
       const [rows]: any = await DB.query(query, [countryId]);
       for (let i = 0; i < rows.length; ++i) {
+        rows[i].country = JSON.parse(rows[i].country);
         rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows[i].isp = JSON.parse(rows[i].isp);
       }
       return rows;
     } catch (e) {
@@ -192,29 +542,63 @@ class NodesApi {
 
   public async $getNodesPerISP(ISPId: string) {
     try {
-      const query = `
-        SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
-          UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
-          geo_names_city.names as city, geo_names_country.names as country
-        FROM node_stats
-        JOIN (
-          SELECT public_key, MAX(added) as last_added
-          FROM node_stats
-          GROUP BY public_key
-        ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
-        RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
-        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+      let query = `
+        SELECT channels.node1_public_key AS node1PublicKey, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1 AND (node1.as_number IN (?) OR node2.as_number IN (?))
+        ORDER BY short_id DESC
+      `;
+
+      const IPSIds = ISPId.split(',');
+      const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
+      if (!rows || rows.length === 0) {
+        return [];
+      }
+
+      const nodes = {};
+
+      const intISPIds: number[] = [];
+      for (const ispId of IPSIds) {
+        intISPIds.push(parseInt(ispId, 10));
+      }
+
+      for (const channel of rows) {
+        if (intISPIds.includes(channel.isp1ID)) {
+          nodes[channel.node1PublicKey] = true;
+        }
+        if (intISPIds.includes(channel.isp2ID)) {
+          nodes[channel.node2PublicKey] = true;
+        }
+      }
+
+      query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
-        WHERE nodes.as_number IN (?)
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        WHERE nodes.public_key IN (?)
         ORDER BY capacity DESC
       `;
 
-      const [rows]: any = await DB.query(query, [ISPId.split(',')]);
-      for (let i = 0; i < rows.length; ++i) {
-        rows[i].country = JSON.parse(rows[i].country);
-        rows[i].city = JSON.parse(rows[i].city);
+      const [rows2]: any = await DB.query(query, [Object.keys(nodes)]);
+      for (let i = 0; i < rows2.length; ++i) {
+        rows2[i].country = JSON.parse(rows2[i].country);
+        rows2[i].city = JSON.parse(rows2[i].city);
+        rows2[i].subdivision = JSON.parse(rows2[i].subdivision);
       }
-      return rows;
+      return rows2;
+
     } catch (e) {
       logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
@@ -227,7 +611,6 @@ class NodesApi {
         FROM nodes
         JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
         JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
-        JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
         GROUP BY country_id
         ORDER BY COUNT(DISTINCT nodes.public_key) DESC
       `;
@@ -253,6 +636,88 @@ class NodesApi {
       throw e;
     }
   }
+
+  /**
+   * Save or update a node present in the graph
+   */
+  public async $saveNode(node: ILightningApi.Node): Promise<void> {
+    try {
+      // https://github.com/mempool/mempool/issues/3006
+      if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
+        node.last_update = null;
+      }
+  
+      const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
+      const query = `INSERT INTO nodes(
+          public_key,
+          first_seen,
+          updated_at,
+          alias,
+          alias_search,
+          color,
+          sockets,
+          status
+        )
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
+        ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
+
+      await DB.query(query, [
+        node.pub_key,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+      ]);
+    } catch (e) {
+      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update node sockets
+   */
+  public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
+    const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
+    try {
+      await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
+    } catch (e) {
+      logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /**
+   * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   */
+   public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
+    if (graphNodesPubkeys.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE nodes
+        SET status = 0
+        WHERE public_key NOT IN (
+          ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
+        )
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private aliasToSearchText(str: string): string {
+    return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '');
+  }
 }
 
 export default new NodesApi();

backend/src/api/explorer/nodes.routes.ts

@@ -2,20 +2,27 @@ import config from '../../config';
 import { Application, Request, Response } from 'express';
 import nodesApi from './nodes.api';
 import DB from '../../database';
+import { INodesRanking } from '../../mempool.interfaces';
 
 class NodesRoutes {
   constructor() { }
 
   public initRoutes(app: Application) {
     app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/liquidity', this.$getTopNodesByCapacity)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/fees/histogram', this.$getFeeHistogram)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
     ;
   }
 
@@ -28,6 +35,96 @@ class NodesRoutes {
     }
   }
 
+  private async $getNodeGroup(req: Request, res: Response) {
+    try {
+      let nodesList;
+      let nodes: any[] = [];
+      switch (config.MEMPOOL.NETWORK) {
+        case 'testnet':
+          nodesList = [
+            '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
+            '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
+            '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
+            '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0',
+            '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3',
+            '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af',
+            '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab',
+            '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8',
+            '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605',
+            '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205',
+            '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18',
+            '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584',
+            '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c',
+            '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5',
+            '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075',
+            '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382',
+            '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1',
+            '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f',
+          ];
+          break;
+        case 'signet':
+          nodesList = [
+            '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
+            '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
+            '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
+            '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029',
+            '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe',
+            '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43',
+            '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991',
+            '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34',
+            '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86',
+            '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7',
+            '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761',
+            '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7',
+            '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070',
+            '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45',
+            '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097',
+            '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853',
+            '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240',
+            '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2',
+          ];
+          break;
+        default:
+          nodesList = [
+            '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
+            '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
+            '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',
+            '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108',
+            '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c',
+            '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5',
+            '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a',
+            '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9',
+            '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34',
+            '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf',
+            '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c',
+            '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43',
+            '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06',
+            '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e',
+            '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7',
+            '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192',
+            '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09',
+            '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57',
+          ];
+      }
+
+      for (let pubKey of nodesList) {
+        try {
+          const node = await nodesApi.$getNode(pubKey);
+          if (node) {
+            nodes.push(node);
+          }
+        } catch (e) {}
+      }
+      
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNode(req: Request, res: Response) {
     try {
       const node = await nodesApi.$getNode(req.params.public_key);
@@ -35,6 +132,9 @@ class NodesRoutes {
         res.status(404).send('Node not found');
         return;
       }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(node);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -44,17 +144,39 @@ class NodesRoutes {
   private async $getHistoricalNodeStats(req: Request, res: Response) {
     try {
       const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(statistics);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
 
-  private async $getTopNodes(req: Request, res: Response) {
+  private async $getFeeHistogram(req: Request, res: Response) {
     try {
-      const topCapacityNodes = await nodesApi.$getTopCapacityNodes();
-      const topChannelsNodes = await nodesApi.$getTopChannelsNodes();
-      res.json({
+      const node = await nodesApi.$getFeeHistogram(req.params.public_key);
+      if (!node) {
+        res.status(404).send('Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);
+      const topChannelsNodes = await nodesApi.$getTopChannelsNodes(false);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(<INodesRanking>{
         topByCapacity: topCapacityNodes,
         topByChannels: topChannelsNodes,
       });
@@ -63,18 +185,45 @@ class NodesRoutes {
     }
   }
 
+  private async $getTopNodesByCapacity(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodesByChannels(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopChannelsNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getOldestNodes(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getOldestNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getISPRanking(req: Request, res: Response): Promise<void> {
     try {
-      const groupBy = req.query.groupBy as string;
-      const showTor = req.query.showTor as string === 'true' ? true : false;
-
-      if (!['capacity', 'node-count'].includes(groupBy)) {
-        res.status(400).send(`groupBy must be one of 'capacity' or 'node-count'`);
-        return;
-      }
-
-      const nodesPerAs = await nodesApi.$getNodesISP(groupBy, showTor);
-
+      const nodesPerAs = await nodesApi.$getNodesISPRanking();
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
@@ -84,6 +233,18 @@ class NodesRoutes {
     }
   }
 
+  private async $getWorldNodes(req: Request, res: Response) {
+    try {
+      const worldNodes = await nodesApi.$getWorldNodes();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(worldNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNodesPerCountry(req: Request, res: Response) {
     try {
       const [country]: any[] = await DB.query(

backend/src/api/explorer/statistics.api.ts

@@ -6,7 +6,8 @@ class StatisticsApi {
   public async $getStatistics(interval: string | null = null): Promise<any> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity, tor_nodes, clearnet_nodes, unannounced_nodes
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
+      tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
       FROM lightning_stats`;
 
     if (interval) {
@@ -27,7 +28,7 @@ class StatisticsApi {
   public async $getLatestStatistics(): Promise<any> {
     try {
       const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
-      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
       return {
         latest: rows[0],
         previous: rows2[0],

backend/src/api/fetch-version.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from "path";
+const { spawnSync } = require('child_process');
+
+function getVersion(): string {
+  const packageJson = fs.readFileSync('package.json').toString();
+  return JSON.parse(packageJson).version;
+}
+
+function getGitCommit(): string {
+  if (process.env.MEMPOOL_COMMIT_HASH) {
+    return process.env.MEMPOOL_COMMIT_HASH;
+  } else {
+    const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+    if (!gitRevParse.error) {
+      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+      if (output) {
+        return output;
+      } else {
+        console.log('Could not fetch git commit: No repo available');
+      }
+    } else if (gitRevParse.error.code === 'ENOENT') {
+      console.log('Could not fetch git commit: Command `git` is unavailable');
+    }
+  }
+  return '?';
+}
+
+const versionInfo = {
+  version: getVersion(),
+  gitCommit: getGitCommit()
+}
+
+fs.writeFileSync(
+  path.join(__dirname, 'version.json'),
+  JSON.stringify(versionInfo, null, 2) + "\n"
+);

backend/src/api/fiat-conversion.ts

@@ -1,123 +0,0 @@
-import logger from '../logger';
-import * as http from 'http';
-import * as https from 'https';
-import axios, { AxiosResponse } from 'axios';
-import { IConversionRates } from '../mempool.interfaces';
-import config from '../config';
-import backendInfo from './backend-info';
-import { SocksProxyAgent } from 'socks-proxy-agent';
-
-class FiatConversion {
-  private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
-    'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
-    'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
-    'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
-  private conversionRates: IConversionRates = {};
-  private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
-  public ratesInitialized = false; // If true, it means rates are ready for use
-
-  constructor() {
-    for (const fiat of this.debasingFiatCurrencies) {
-      this.conversionRates[fiat] = 0;
-    }
-  }
-
-  public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
-    this.ratesChangedCallback = fn;
-  }
-
-  public startService() {
-    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
-    logger.info('Starting currency rates service');
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
-    } else {
-      logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
-    }
-    setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
-    this.updateCurrency();
-  }
-
-  public getConversionRates() {
-    return this.conversionRates;
-  }
-
-  private async updateCurrency(): Promise<void> {
-    type axiosOptions = {
-      headers: {
-        'User-Agent': string
-      };
-      timeout: number;
-      httpAgent?: http.Agent;
-      httpsAgent?: https.Agent;
-    }
-    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
-    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
-    const axiosOptions: axiosOptions = {
-      headers: {
-        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
-      },
-      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
-    };
-
-    let retry = 0;
-
-    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-      try {
-        if (config.SOCKS5PROXY.ENABLED) {
-          let socksOptions: any = {
-            agentOptions: {
-              keepAlive: true,
-            },
-            hostname: config.SOCKS5PROXY.HOST,
-            port: config.SOCKS5PROXY.PORT
-          };
-
-          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-            socksOptions.username = config.SOCKS5PROXY.USERNAME;
-            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-          } else {
-            // Retry with different tor circuits https://stackoverflow.com/a/64960234
-            socksOptions.username = `circuit${retry}`;
-          }
-
-          // Handle proxy agent for onion addresses
-          if (isHTTP) {
-            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
-          } else {
-            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-          }
-        }
-        
-        logger.debug('Querying currency rates service...');
-
-        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
-
-        if (response.statusText === 'error' || !response.data) {
-          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
-        }
-
-        for (const rate of response.data.data) {
-          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
-            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
-          }
-        }
-
-        this.ratesInitialized = true;
-        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
-
-        if (this.ratesChangedCallback) {
-          this.ratesChangedCallback(this.conversionRates);
-        }
-        break;
-      } catch (e) {
-        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
-        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
-        retry++;
-      }
-    }
-  }
-}
-
-export default new FiatConversion();

backend/src/api/lightning/clightning/clightning-client.ts

@@ -0,0 +1,270 @@
+// Imported from https://github.com/shesek/lightning-client-js
+
+'use strict';
+
+const methods = [
+  'addgossip',
+  'autocleaninvoice',
+  'check',
+  'checkmessage',
+  'close',
+  'connect',
+  'createinvoice',
+  'createinvoicerequest',
+  'createoffer',
+  'createonion',
+  'decode',
+  'decodepay',
+  'delexpiredinvoice',
+  'delinvoice',
+  'delpay',
+  'dev-listaddrs',
+  'dev-rescan-outputs',
+  'disableoffer',
+  'disconnect',
+  'estimatefees',
+  'feerates',
+  'fetchinvoice',
+  'fundchannel',
+  'fundchannel_cancel',
+  'fundchannel_complete',
+  'fundchannel_start',
+  'fundpsbt',
+  'getchaininfo',
+  'getinfo',
+  'getlog',
+  'getrawblockbyheight',
+  'getroute',
+  'getsharedsecret',
+  'getutxout',
+  'help',
+  'invoice',
+  'keysend',
+  'legacypay',
+  'listchannels',
+  'listconfigs',
+  'listforwards',
+  'listfunds',
+  'listinvoices',
+  'listnodes',
+  'listoffers',
+  'listpays',
+  'listpeers',
+  'listsendpays',
+  'listtransactions',
+  'multifundchannel',
+  'multiwithdraw',
+  'newaddr',
+  'notifications',
+  'offer',
+  'offerout',
+  'openchannel_abort',
+  'openchannel_bump',
+  'openchannel_init',
+  'openchannel_signed',
+  'openchannel_update',
+  'pay',
+  'payersign',
+  'paystatus',
+  'ping',
+  'plugin',
+  'reserveinputs',
+  'sendinvoice',
+  'sendonion',
+  'sendonionmessage',
+  'sendpay',
+  'sendpsbt',
+  'sendrawtransaction',
+  'setchannelfee',
+  'signmessage',
+  'signpsbt',
+  'stop',
+  'txdiscard',
+  'txprepare',
+  'txsend',
+  'unreserveinputs',
+  'utxopsbt',
+  'waitanyinvoice',
+  'waitblockheight',
+  'waitinvoice',
+  'waitsendpay',
+  'withdraw'
+];
+
+
+import EventEmitter from 'events';
+import { existsSync, statSync } from 'fs';
+import { createConnection, Socket } from 'net';
+import { homedir } from 'os';
+import path from 'path';
+import { createInterface, Interface } from 'readline';
+import logger from '../../../logger';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
+
+class LightningError extends Error {
+  type: string = 'lightning';
+  message: string = 'lightning-client error';
+
+  constructor(error) {
+    super();
+    this.type = error.type;
+    this.message = error.message;
+  }
+}
+
+const defaultRpcPath = path.join(homedir(), '.lightning')
+  , fStat = (...p) => statSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p))
+
+export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
+  private rpcPath: string;
+  private reconnectWait: number;
+  private reconnectTimeout;
+  private reqcount: number;
+  private client: Socket;
+  private rl: Interface;
+  private clientConnectionPromise: Promise<unknown>;
+
+  constructor(rpcPath = defaultRpcPath) {
+    if (!path.isAbsolute(rpcPath)) {
+      throw new Error('The rpcPath must be an absolute path');
+    }
+
+    if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
+      // network directory provided, use the lightning-rpc within in
+      if (fExists(rpcPath, 'lightning-rpc')) {
+        rpcPath = path.join(rpcPath, 'lightning-rpc');
+      }
+
+      // main data directory provided, default to using the bitcoin mainnet subdirectory
+      // to be removed in v0.2.0
+      else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
+        logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln)
+        logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln)
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+      }
+    }
+
+    logger.debug(`Connecting to ${rpcPath}`, logger.tags.ln);
+
+    super();
+    this.rpcPath = rpcPath;
+    this.reconnectWait = 0.5;
+    this.reconnectTimeout = null;
+    this.reqcount = 0;
+
+    const _self = this;
+
+    this.client = createConnection(rpcPath).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+    this.rl = createInterface({ input: this.client }).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+
+    this.clientConnectionPromise = new Promise<void>(resolve => {
+      _self.client.on('connect', () => {
+        logger.info(`CLightning client connected`, logger.tags.ln);
+        _self.reconnectWait = 1;
+        resolve();
+      });
+
+      _self.client.on('end', () => {
+        logger.err(`CLightning client connection closed, reconnecting`, logger.tags.ln);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+
+      _self.client.on('error', error => {
+        logger.err(`CLightning client connection error: ${error}`, logger.tags.ln);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+    });
+
+    this.rl.on('line', line => {
+      line = line.trim();
+      if (!line) {
+        return;
+      }
+      const data = JSON.parse(line);
+      _self.emit('res:' + data.id, data);
+    });
+  }
+
+  increaseWaitTime(): void {
+    if (this.reconnectWait >= 16) {
+      this.reconnectWait = 16;
+    } else {
+      this.reconnectWait *= 2;
+    }
+  }
+
+  reconnect(): void {
+    const _self = this;
+
+    if (this.reconnectTimeout) {
+      return;
+    }
+
+    this.reconnectTimeout = setTimeout(() => {
+      logger.debug(`Trying to reconnect...`, logger.tags.ln);
+
+      _self.client.connect(_self.rpcPath);
+      _self.reconnectTimeout = null;
+    }, this.reconnectWait * 1000);
+  }
+
+  call(method, args = []): Promise<any> {
+    const _self = this;
+
+    const callInt = ++this.reqcount;
+    const sendObj = {
+      jsonrpc: '2.0',
+      method,
+      params: args,
+      id: '' + callInt
+    };
+
+
+    // Wait for the client to connect
+    return this.clientConnectionPromise
+      .then(() => new Promise((resolve, reject) => {
+        // Wait for a response
+        this.once('res:' + callInt, res => res.error == null
+          ? resolve(res.result)
+          : reject(new LightningError(res.error))
+        );
+
+        // Send the command
+        _self.client.write(JSON.stringify(sendObj));
+      }));
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const listnodes: any[] = await this.call('listnodes');
+    const listchannels: any[] = await this.call('listchannels');
+    const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
+
+    return {
+      nodes: listnodes['nodes'].map(node => convertNode(node)),
+      edges: channelsList,
+    };
+  }
+}
+
+const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
+
+methods.forEach(k => {
+  CLightningClient.prototype[protify(k)] = function (...args: any) {
+    return this.call(k, args);
+  };
+});

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -0,0 +1,158 @@
+import { ILightningApi } from '../lightning-api.interface';
+import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
+import logger from '../../../logger';
+import { Common } from '../../common';
+import config from '../../../config';
+
+/**
+ * Convert a clightning "listnode" entry to a lnd node entry
+ */
+export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
+  return {
+    alias: clNode.alias ?? '',
+    color: `#${clNode.color ?? ''}`,
+    features: [], // TODO parse and return clNode.feature
+    pub_key: clNode.nodeid,
+    addresses: clNode.addresses?.map((addr) => {
+      let address = addr.address;
+      if (addr.type === 'ipv6') {
+        address = `[${address}]`;
+      }
+      return {
+        network: addr.type,
+        addr: `${address}:${addr.port}`
+      };
+    }) ?? [],
+    last_update: clNode?.last_timestamp ?? 0,
+    custom_records
+  };
+}
+
+/**
+ * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ */
+export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
+  logger.debug(`Converting clightning nodes and channels to lnd graph format`, logger.tags.ln);
+
+  let loggerTimer = new Date().getTime() / 1000;
+  let channelProcessed = 0;
+
+  const consolidatedChannelList: ILightningApi.Channel[] = [];
+  const clChannelsDict = {};
+  const clChannelsDictCount = {};
+
+  for (const clChannel of clChannels) {
+    if (!clChannelsDict[clChannel.short_channel_id]) {
+      clChannelsDict[clChannel.short_channel_id] = clChannel;
+      clChannelsDictCount[clChannel.short_channel_id] = 1;
+    } else {
+      const fullChannel = await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]);
+      if (fullChannel !== null) {
+        consolidatedChannelList.push(fullChannel);
+        delete clChannelsDict[clChannel.short_channel_id];
+        clChannelsDictCount[clChannel.short_channel_id]++;
+      }
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`, logger.tags.ln);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    ++channelProcessed;
+  }
+
+  channelProcessed = 0;
+  const keys = Object.keys(clChannelsDict);
+  for (const short_channel_id of keys) {
+    const incompleteChannel = await buildIncompleteChannel(clChannelsDict[short_channel_id]);
+    if (incompleteChannel !== null) {
+      consolidatedChannelList.push(incompleteChannel);
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+      logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    channelProcessed++;
+  }
+
+  return consolidatedChannelList;
+}
+
+/**
+ * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy for both nodes
+ */
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel | null> {
+  const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
+
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
+  if (!tx) {
+    return null;
+  }
+  const parts = clChannelA.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
+    capacity: clChannelA.satoshis,
+    last_update: lastUpdate,
+    node1_policy: convertPolicy(clChannelA),
+    node2_policy: convertPolicy(clChannelB),
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannelA.source,
+    node2_pub: clChannelB.source,
+  };
+}
+
+/**
+ * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy of only one node
+ */
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel | null> {
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
+  if (!tx) {
+    return null;
+  }
+  const parts = clChannel.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
+    capacity: clChannel.satoshis,
+    last_update: clChannel.last_update ?? 0,
+    node1_policy: convertPolicy(clChannel),
+    node2_policy: null,
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannel.source,
+    node2_pub: clChannel.destination,
+  };
+}
+
+/**
+ * Convert a clightning "listnode" response to a lnd channel policy format
+ */
+function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
+  return {
+    time_lock_delta: clChannel.delay,
+    min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
+    max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
+    fee_base_msat: clChannel.base_fee_millisatoshi,
+    fee_rate_milli_msat: clChannel.fee_per_millionth,
+    disabled: !clChannel.active,
+    last_update: clChannel.last_update ?? 0,
+  };
+}

backend/src/api/lightning/lightning-api-abstract-factory.ts

@@ -1,7 +1,5 @@
 import { ILightningApi } from './lightning-api.interface';
 
 export interface AbstractLightningApi {
-  $getNetworkInfo(): Promise<ILightningApi.NetworkInfo>;
   $getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
-  $getInfo(): Promise<ILightningApi.Info>;
 }

backend/src/api/lightning/lightning-api-factory.ts

@@ -1,9 +1,12 @@
 import config from '../../config';
+import CLightningClient from './clightning/clightning-client';
 import { AbstractLightningApi } from './lightning-api-abstract-factory';
 import LndApi from './lnd/lnd-api';
 
 function lightningApiFactory(): AbstractLightningApi {
-  switch (config.LIGHTNING.BACKEND) {
+  switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
+    case 'cln':
+      return new CLightningClient(config.CLIGHTNING.SOCKET);
     case 'lnd':
     default:
       return new LndApi();

backend/src/api/lightning/lightning-api.interface.ts

@@ -1,71 +1,92 @@
 export namespace ILightningApi {
   export interface NetworkInfo {
-    average_channel_size: number;
-    channel_count: number;
-    max_channel_size: number;
-    median_channel_size: number;
-    min_channel_size: number;
-    node_count: number;
-    not_recently_updated_policy_count: number;
-    total_capacity: number;
+    graph_diameter: number;
+    avg_out_degree: number;
+    max_out_degree: number;
+    num_nodes: number;
+    num_channels: number;
+    total_network_capacity: string;
+    avg_channel_size: number;
+    min_channel_size: string;
+    max_channel_size: string;
+    median_channel_size_sat: string;
+    num_zombie_chans: string;
   }
 
   export interface NetworkGraph {
-    channels: Channel[];
     nodes: Node[];
+    edges: Channel[];
   }
 
   export interface Channel {
-    id: string;
-    capacity: number;
-    policies: Policy[];
-    transaction_id: string;
-    transaction_vout: number;
-    updated_at?: string;
+    channel_id: string;
+    chan_point: string;
+    last_update: number | null;
+    node1_pub: string;
+    node2_pub: string;
+    capacity: string;
+    node1_policy: RoutingPolicy | null;
+    node2_policy: RoutingPolicy | null;
   }
 
-  interface Policy {
-    public_key: string;
-    base_fee_mtokens?: string;
-    cltv_delta?: number;
-    fee_rate?: number;
-    is_disabled?: boolean;
-    max_htlc_mtokens?: string;
-    min_htlc_mtokens?: string;
-    updated_at?: string;
+  export interface RoutingPolicy {
+    time_lock_delta: number;
+    min_htlc: string;
+    fee_base_msat: string;
+    fee_rate_milli_msat: string;
+    disabled: boolean;
+    max_htlc_msat: string;
+    last_update: number | null;
   }
 
   export interface Node {
+    last_update: number | null;
+    pub_key: string;
     alias: string;
+    addresses: {
+      network: string;
+      addr: string;
+    }[];
     color: string;
-    features: Feature[];
-    public_key: string;
-    sockets: string[];
-    updated_at?: string;
+    features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
   }
 
   export interface Info {
-    chains: string[];
-    color: string;
-    active_channels_count: number;
+    identity_pubkey: string;
     alias: string;
-    current_block_hash: string;
-    current_block_height: number;
-    features: Feature[];
-    is_synced_to_chain: boolean;
-    is_synced_to_graph: boolean;
-    latest_block_at: string;
-    peers_count: number;
-    pending_channels_count: number;
-    public_key: string;
-    uris: any[];
+    num_pending_channels: number;
+    num_active_channels: number;
+    num_peers: number;
+    block_height: number;
+    block_hash: string;
+    synced_to_chain: boolean;
+    testnet: boolean;
+    uris: string[];
+    best_header_timestamp: string;
     version: string;
+    num_inactive_channels: number;
+    chains: {
+      chain: string;
+      network: string;
+    }[];
+    color: string;
+    synced_to_graph: boolean;
+    features: { [key: number]: Feature };
+    commit_hash: string;
+    /** Available on LND since v0.15.0-beta */
+    require_htlc_interceptor?: boolean;
   }
-  
+
   export interface Feature {
-    bit: number;
-    is_known: boolean;
+    name: string;
     is_required: boolean;
-    type?: string;
+    is_known: boolean;
   }
-}
+
+  export interface ForensicOutput {
+    node?: 1 | 2;
+    type: number;
+    value: number;
+  }
+}

backend/src/api/lightning/lnd/lnd-api.ts

@@ -1,44 +1,40 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { Agent } from 'https';
+import * as fs from 'fs';
 import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
-import * as fs from 'fs';
-import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning';
 import config from '../../../config';
-import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
-  private lnd: any;
+  axiosConfig: AxiosRequestConfig = {};
+
   constructor() {
-    if (!config.LIGHTNING.ENABLED) {
-      return;
-    }
-    try {
-      const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64');
-      const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64');
-
-      const { lnd } = authenticatedLndGrpc({
-        cert: tls,
-        macaroon: macaroon,
-        socket: config.LND.SOCKET,
-      });
-
-      this.lnd = lnd;
-    } catch (e) {
-      logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
-      process.exit(1);
+    if (config.LIGHTNING.ENABLED) {
+      this.axiosConfig = {
+        headers: {
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
+        },
+        httpsAgent: new Agent({
+          ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
+        }),
+        timeout: 10000
+      };
     }
   }
 
   async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
-    return await getNetworkInfo({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getInfo(): Promise<ILightningApi.Info> {
-    // @ts-ignore
-    return await getWalletInfo({ lnd: this.lnd });
+    return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
-    return await getNetworkGraph({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+      .then((response) => response.data);
   }
 }
 

backend/src/api/mempool-blocks.ts

@@ -1,11 +1,14 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, ThreadTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
+import { Worker } from 'worker_threads';
+import path from 'path';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
   private mempoolBlockDeltas: MempoolBlockDelta[] = [];
+  private txSelectionWorker: Worker | null = null;
 
   constructor() {}
 
@@ -30,7 +33,7 @@ class MempoolBlocks {
     return this.mempoolBlockDeltas;
   }
 
-  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
+  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
     const latestMempool = memPool;
     const memPoolArray: TransactionExtended[] = [];
     for (const i in latestMempool) {
@@ -71,15 +74,19 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
-    this.mempoolBlocks = blocks;
-    this.mempoolBlockDeltas = deltas;
+    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
+      this.mempoolBlocks = blocks;
+      this.mempoolBlockDeltas = deltas;
+    }
+
+    return blocks;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
-    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     let blockWeight = 0;
     let blockSize = 0;
     let transactions: TransactionExtended[] = [];
@@ -90,16 +97,21 @@ class MempoolBlocks {
         blockSize += tx.size;
         transactions.push(tx);
       } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
         blockWeight = tx.weight;
         blockSize = tx.size;
         transactions = [tx];
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
     }
-    // Calculate change from previous block states
+
+    return mempoolBlocks;
+  }
+
+  private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
       let removed: string[] = [];
@@ -132,14 +144,166 @@ class MempoolBlocks {
         removed
       });
     }
-    return {
-      blocks: mempoolBlocks,
-      deltas: mempoolBlockDeltas
-    };
+    return mempoolBlockDeltas;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[],
-    blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
+  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const strippedMempool: { [txid: string]: ThreadTransaction } = {};
+    Object.values(newMempool).forEach(entry => {
+      strippedMempool[entry.txid] = {
+        txid: entry.txid,
+        fee: entry.fee,
+        weight: entry.weight,
+        feePerVsize: entry.fee / (entry.weight / 4),
+        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
+        vin: entry.vin.map(v => v.txid),
+      };
+    });
+
+    // (re)initialize tx selection worker thread
+    if (!this.txSelectionWorker) {
+      this.txSelectionWorker = new Worker(path.resolve(__dirname, './tx-selection-worker.js'));
+      // if the thread throws an unexpected error, or exits for any other reason,
+      // reset worker state so that it will be re-initialized on the next run
+      this.txSelectionWorker.once('error', () => {
+        this.txSelectionWorker = null;
+      });
+      this.txSelectionWorker.once('exit', () => {
+        this.txSelectionWorker = null;
+      });
+    }
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
+      const { blocks, clusters } = await workerResultPromise;
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+
+      return this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
+    } catch (e) {
+      logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+    return this.mempoolBlocks;
+  }
+
+  public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
+    if (!this.txSelectionWorker) {
+      // need to reset the worker
+      this.makeBlockTemplates(newMempool, saveResults);
+      return;
+    }
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const addedStripped: ThreadTransaction[] = added.map(entry => {
+      return {
+        txid: entry.txid,
+        fee: entry.fee,
+        weight: entry.weight,
+        feePerVsize: entry.fee / (entry.weight / 4),
+        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
+        vin: entry.vin.map(v => v.txid),
+      };
+    });
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
+      const { blocks, clusters } = await workerResultPromise;
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+
+      this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
+    } catch (e) {
+      logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] {
+    // update this thread's mempool with the results
+    blocks.forEach(block => {
+      block.forEach(tx => {
+        if (tx.txid in mempool) {
+          if (tx.effectiveFeePerVsize != null) {
+            mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+          }
+          if (tx.cpfpRoot && tx.cpfpRoot in clusters) {
+            const ancestors: Ancestor[] = [];
+            const descendants: Ancestor[] = [];
+            const cluster = clusters[tx.cpfpRoot];
+            let matched = false;
+            cluster.forEach(txid => {
+              if (txid === tx.txid) {
+                matched = true;
+              } else {
+                const relative = {
+                  txid: txid,
+                  fee: mempool[txid].fee,
+                  weight: mempool[txid].weight,
+                };
+                if (matched) {
+                  descendants.push(relative);
+                } else {
+                  ancestors.push(relative);
+                }
+              }
+            });
+            mempool[tx.txid].ancestors = ancestors;
+            mempool[tx.txid].descendants = descendants;
+            mempool[tx.txid].bestDescendant = null;
+          }
+          mempool[tx.txid].cpfpChecked = tx.cpfpChecked;
+        }
+      });
+    });
+
+    // unpack the condensed blocks into proper mempool blocks
+    const mempoolBlocks = blocks.map((transactions, blockIndex) => {
+      return this.dataToMempoolBlocks(transactions.map(tx => {
+        return mempool[tx.txid] || null;
+      }).filter(tx => !!tx), blockIndex);
+    });
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
+      this.mempoolBlocks = mempoolBlocks;
+      this.mempoolBlockDeltas = deltas;
+    }
+
+    return mempoolBlocks;
+  }
+
+  private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
+    let totalSize = 0;
+    let totalWeight = 0;
+    const fitTransactions: TransactionExtended[] = [];
+    transactions.forEach(tx => {
+      totalSize += tx.size;
+      totalWeight += tx.weight;
+      if ((totalWeight + tx.weight) <= config.MEMPOOL.BLOCK_WEIGHT_UNITS * 1.2) {
+        fitTransactions.push(tx);
+      }
+    });
     let rangeLength = 4;
     if (blocksIndex === 0) {
       rangeLength = 8;
@@ -150,14 +314,14 @@ class MempoolBlocks {
       rangeLength = 8;
     }
     return {
-      blockSize: blockSize,
-      blockVSize: blockWeight / 4,
+      blockSize: totalSize,
+      blockVSize: totalWeight / 4,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
       medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
-      transactions: transactions.map((tx) => Common.stripTransaction(tx)),
+      transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
 }

backend/src/api/mempool.ts

@@ -20,6 +20,8 @@ class Mempool {
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
+  private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+    deletedTransactions: TransactionExtended[]) => Promise<void>) | undefined;
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
@@ -29,6 +31,11 @@ class Mempool {
   private mempoolProtection = 0;
   private latestTransactions: any[] = [];
 
+  private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100; 
+  private SAMPLE_TIME = 10000; // In ms
+  private timer = new Date().getTime();
+  private missingTxCount = 0;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
     setInterval(this.deleteExpiredTransactions.bind(this), 20000);
@@ -63,6 +70,11 @@ class Mempool {
     this.mempoolChangedCallback = fn;
   }
 
+  public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise<void>) {
+    this.asyncMempoolChangedCallback = fn;
+  }
+
   public getMempool(): { [txid: string]: TransactionExtended } {
     return this.mempoolCache;
   }
@@ -72,6 +84,9 @@ class Mempool {
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
     }
+    if (this.asyncMempoolChangedCallback) {
+      this.asyncMempoolChangedCallback(this.mempoolCache, [], []);
+    }
   }
 
   public async $updateMemPoolInfo() {
@@ -103,12 +118,11 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
-    logger.debug('Updating mempool');
+  public async $updateMempool(): Promise<void> {
+    logger.debug(`Updating mempool...`);
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
     const transactions = await bitcoinApi.$getRawMempool();
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
@@ -119,12 +133,21 @@ class Mempool {
       loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
     }
 
+    // https://github.com/mempool/mempool/issues/3283
+    const logEsplora404 = (missingTxCount, threshold, time) => {
+      const log = `In the past ${time / 1000} seconds, esplora tx API replied ${missingTxCount} times with a 404 error code while updating nodejs backend mempool`;
+      if (missingTxCount >= threshold) {
+        logger.warn(log);
+      } else if (missingTxCount > 0) {
+        logger.debug(log);
+      }
+    };
+
     for (const txid of transactions) {
       if (!this.mempoolCache[txid]) {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
           this.mempoolCache[txid] = transaction;
-          txCount++;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -133,14 +156,12 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
-        } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+        } catch (e: any) {
+          if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
+            this.missingTxCount++;
+          }
+          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
 
@@ -149,6 +170,14 @@ class Mempool {
       }
     }
 
+    // Reset esplora 404 counter and log a warning if needed
+    const elapsedTime = new Date().getTime() - this.timer;
+    if (elapsedTime > this.SAMPLE_TIME) {
+      logEsplora404(this.missingTxCount, this.ESPLORA_MISSING_TX_WARNING_THRESHOLD, elapsedTime);
+      this.timer = new Date().getTime();
+      this.missingTxCount = 0;
+    }
+
     // Prevent mempool from clear on bitcoind restart by delaying the deletion
     if (this.mempoolProtection === 0
       && currentMempoolSize > 20000
@@ -194,18 +223,20 @@ class Mempool {
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
+    if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    }
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
     for (const rbfTransaction in rbfTransactions) {
       if (this.mempoolCache[rbfTransaction]) {
         // Store replaced transactions
-        rbfCache.add(rbfTransaction, rbfTransactions[rbfTransaction].txid);
+        rbfCache.add(this.mempoolCache[rbfTransaction], rbfTransactions[rbfTransaction].txid);
         // Erase the replaced transactions from the local mempool
         delete this.mempoolCache[rbfTransaction];
       }
@@ -231,6 +262,7 @@ class Mempool {
       const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
       if (lazyDeleteAt && lazyDeleteAt < now) {
         delete this.mempoolCache[tx];
+        rbfCache.evict(tx);
       }
     }
   }

backend/src/api/mining/mining-routes.ts

@@ -7,6 +7,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
 import HashratesRepository from '../../repositories/HashratesRepository';
 import bitcoinClient from '../bitcoin/bitcoin-client';
 import mining from "./mining";
+import PricesRepository from '../../repositories/PricesRepository';
 
 class MiningRoutes {
   public initRoutes(app: Application) {
@@ -26,10 +27,32 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
     ;
   }
 
+  private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
+    try {
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      if (req.query.timestamp) {
+        res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
+          parseInt(<string>req.query.timestamp ?? 0, 10)
+        ));
+      } else {
+        res.status(200).send(await PricesRepository.$getHistoricalPrices());
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getPool(req: Request, res: Response): Promise<void> {
     try {
       const stats = await mining.$getPoolStat(req.params.slug);
@@ -238,6 +261,12 @@ class MiningRoutes {
   public async $getBlockAudit(req: Request, res: Response) {
     try {
       const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
+
+      if (!audit) {
+        res.status(204).send(`This block has not been audited.`);
+        return;
+      }
+
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
@@ -246,6 +275,55 @@ class MiningRoutes {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
+
+  private async $getHeightFromTimestamp(req: Request, res: Response) {
+    try {
+      const timestamp = parseInt(req.params.timestamp, 10);
+      // This will prevent people from entering milliseconds etc.
+      // Block timestamps are allowed to be up to 2 hours off, so 24 hours
+      // will never put the maximum value before the most recent block
+      const nowPlus1day = Math.floor(Date.now() / 1000) + 60 * 60 * 24;
+      // Prevent non-integers that are not seconds
+      if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp > nowPlus1day) {
+        throw new Error(`Invalid timestamp, value must be Unix seconds`);
+      }
+      const result = await BlocksRepository.$getBlockHeightFromTimestamp(
+        timestamp,
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getBlockAuditScores(req: Request, res: Response) {
+    try {
+      let height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+      if (height == null) {
+        height = await BlocksRepository.$mostRecentBlockHeight();
+      }
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await BlocksAuditsRepository.$getBlockAuditScores(height, height - 15));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAuditScore(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash);
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit || 'null');
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 }
 
 export default new MiningRoutes();

backend/src/api/mining/mining.ts

@@ -11,12 +11,14 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
 import config from '../../config';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import PricesRepository from '../../repositories/PricesRepository';
+import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
+import { IEsploraApi } from '../bitcoin/esplora-api.interface';
+import database from '../../database';
 
 class Mining {
-  blocksPriceIndexingRunning = false;
-
-  constructor() {
-  }
+  private blocksPriceIndexingRunning = false;
+  public lastHashrateIndexingDate: number | null = null;
+  public lastWeeklyHashrateIndexingDate: number | null = null;
 
   /**
    * Get historical block predictions match rate
@@ -100,6 +102,7 @@ class Mining {
         rank: rank++,
         emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
         slug: poolInfo.slug,
+        avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
       };
       poolsStats.push(poolStat);
     });
@@ -115,7 +118,7 @@ class Mining {
       poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
     } catch (e) {
       poolsStatistics['lastEstimatedHashrate'] = 0;
-      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
     }
 
     return poolsStatistics;
@@ -139,11 +142,14 @@ class Mining {
     const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
     const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
 
+    const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);    
+    const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);    
+
     let currentEstimatedHashrate = 0;
     try {
       currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
     } catch (e) {
-      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
     }
 
     return {
@@ -160,6 +166,8 @@ class Mining {
       },
       estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
       reportedHashrate: null,
+      avgBlockHealth: avgHealth,
+      totalReward: totalReward,
     };
   }
 
@@ -171,25 +179,26 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate weekly mining pool hashrate history
+   * Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
-    const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
 
     // Run only if:
-    // * lastestRunDate is set to 0 (node backend restart, reorg)
+    // * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg)
     // * we started a new week (around Monday midnight)
-    const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
+    const runIndexing = this.lastWeeklyHashrateIndexingDate === null ||
+      now.getUTCDay() === 1 && this.lastWeeklyHashrateIndexingDate !== now.getUTCDate();
     if (!runIndexing) {
+      logger.debug(`Pool hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
       return;
     }
 
     try {
       const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
-      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.time * 1000;
+      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
 
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
       const hashrates: any[] = [];
@@ -205,7 +214,7 @@ class Mining {
       const startedAt = new Date().getTime() / 1000;
       let timer = new Date().getTime() / 1000;
 
-      logger.debug(`Indexing weekly mining pool hashrate`);
+      logger.debug(`Indexing weekly mining pool hashrate`, logger.tags.mining);
       loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -242,7 +251,7 @@ class Mining {
             });
           }
 
-          newlyIndexed += hashrates.length;
+          newlyIndexed += hashrates.length / Math.max(1, pools.length);
           await HashratesRepository.$saveHashrates(hashrates);
           hashrates.length = 0;
         }
@@ -253,7 +262,7 @@ class Mining {
           const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
           const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
           loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
@@ -263,36 +272,36 @@ class Mining {
         ++indexedThisRun;
         ++totalIndexed;
       }
-      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
+      this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate();
       if (newlyIndexed > 0) {
-        logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
+        logger.info(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
       } else {
-        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
+        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
       }
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
-      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
       throw e;
     }
   }
 
   /**
-   * [INDEXING] Generate daily hashrate data
+   * Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // We only run this once a day around midnight
-    const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
-    const now = new Date().getUTCDate();
-    if (now === latestRunDate) {
+    const today = new Date().getUTCDate();
+    if (today === this.lastHashrateIndexingDate) {
+      logger.debug(`Network hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
       return;
     }
 
     const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
     try {
-      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.time * 1000;
+      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
       const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       const lastMidnight = this.getDateMidnight(new Date());
       let toTimestamp = Math.round(lastMidnight.getTime());
@@ -305,7 +314,7 @@ class Mining {
       const startedAt = new Date().getTime() / 1000;
       let timer = new Date().getTime() / 1000;
 
-      logger.debug(`Indexing daily network hashrate`);
+      logger.debug(`Indexing daily network hashrate`, logger.tags.mining);
       loadingIndicators.setProgress('daily-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -343,7 +352,7 @@ class Mining {
           const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
           const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
           loadingIndicators.setProgress('daily-hashrate-indexing', progress);
@@ -368,16 +377,16 @@ class Mining {
       newlyIndexed += hashrates.length;
       await HashratesRepository.$saveHashrates(hashrates);
 
-      await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
+      this.lastHashrateIndexingDate = new Date().getUTCDate();
       if (newlyIndexed > 0) {
-        logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
+        logger.info(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
       } else {
-        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
+        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
       }
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
-      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Daily network hashrate indexing failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
       throw e;
     }
   }
@@ -393,13 +402,13 @@ class Mining {
     }
 
     const blocks: any = await BlocksRepository.$getBlocksDifficulty();
-    const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+    const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
     let currentDifficulty = genesisBlock.difficulty;
     let totalIndexed = 0;
 
     if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
       await DifficultyAdjustmentsRepository.$saveAdjustments({
-        time: genesisBlock.time,
+        time: genesisBlock.timestamp,
         height: 0,
         difficulty: currentDifficulty,
         adjustment: 0.0,
@@ -443,22 +452,22 @@ class Mining {
       const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
       if (elapsedSeconds > 5) {
         const progress = Math.round(totalBlockChecked / blocks.length * 100);
-        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`);
+        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
         timer = new Date().getTime() / 1000;
       }
     }
 
     if (totalIndexed > 0) {
-      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
+      logger.info(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
     } else {
-      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`);
+      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
     }
   }
 
   /**
    * Create a link between blocks and the latest price at when they were mined
    */
-  public async $indexBlockPrices() {
+  public async $indexBlockPrices(): Promise<void> {
     if (this.blocksPriceIndexingRunning === true) {
       return;
     }
@@ -473,7 +482,7 @@ class Mining {
 
       for (const block of blocksWithoutPrices) {
         // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
-        if (block.height < 68951) {
+        if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
           blocksPrices.push({
             height: block.height,
             priceId: prices[0].id,
@@ -492,11 +501,11 @@ class Mining {
 
         if (blocksPrices.length >= 100000) {
           totalInserted += blocksPrices.length;
+          let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
           if (blocksWithoutPrices.length > 200000) {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-          } else {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+            logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
           }
+          logger.debug(logStr, logger.tags.mining);
           await BlocksRepository.$saveBlockPrices(blocksPrices);
           blocksPrices.length = 0;
         }
@@ -504,11 +513,11 @@ class Mining {
 
       if (blocksPrices.length > 0) {
         totalInserted += blocksPrices.length;
+        let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
         if (blocksWithoutPrices.length > 200000) {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-        } else {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+          logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
         }
+        logger.debug(logStr, logger.tags.mining);
         await BlocksRepository.$saveBlockPrices(blocksPrices);
       }
     } catch (e) {
@@ -519,6 +528,41 @@ class Mining {
     this.blocksPriceIndexingRunning = false;
   }
 
+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);
@@ -530,6 +574,7 @@ class Mining {
 
   private getTimeRange(interval: string | null, scale = 1): number {
     switch (interval) {
+      case '4y': return 43200 * scale; // 12h
       case '3y': return 43200 * scale; // 12h
       case '2y': return 28800 * scale; // 8h
       case '1y': return 28800 * scale; // 8h

backend/src/api/pools-parser.ts

@@ -1,251 +1,161 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
-import BlocksRepository from '../repositories/BlocksRepository';
-
-interface Pool {
-  name: string;
-  link: string;
-  regexes: string[];
-  addresses: string[];
-  slug: string;
-}
+import PoolsRepository from '../repositories/PoolsRepository';
+import { PoolTag } from '../mempool.interfaces';
+import diskCache from './disk-cache';
 
 class PoolsParser {
   miningPools: any[] = [];
   unknownPool: any = {
-    'name': "Unknown",
-    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
-    'regexes': "[]",
-    'addresses': "[]",
+    'id': 0,
+    'name': 'Unknown',
+    'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
+    'regexes': '[]',
+    'addresses': '[]',
     'slug': 'unknown'
   };
-  slugWarnFlag = false;
+  private uniqueLogs: string[] = [];
+
+  private uniqueLog(loggerFunction: any, msg: string): void {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
+  public setMiningPools(pools): void {
+    for (const pool of pools) {
+      pool.regexes = pool.tags;
+      pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+      delete(pool.tags);
+    }
+    this.miningPools = pools;
+  }
 
   /**
-   * Parse the pools.json file, consolidate the data and dump it into the database
+   * Populate our db with updated mining pool definition
+   * @param pools 
    */
-  public async migratePoolsJson(poolsJson: object) {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return;
-    }
+  public async migratePoolsJson(): Promise<void> {
+    // We also need to wipe the backend cache to make sure we don't serve blocks with
+    // the wrong mining pool (usually happen with unknown blocks)
+    diskCache.wipeCache();
 
-    // First we save every entries without paying attention to pool duplication
-    const poolsDuplicated: Pool[] = [];
+    await this.$insertUnknownPool();
 
-    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
-    for (let i = 0; i < coinbaseTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>coinbaseTags[i][1]).name,
-        'link': (<Pool>coinbaseTags[i][1]).link,
-        'regexes': [coinbaseTags[i][0]],
-        'addresses': [],
-        'slug': ''
-      });
-    }
-    const addressesTags = Object.entries(poolsJson['payout_addresses']);
-    for (let i = 0; i < addressesTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>addressesTags[i][1]).name,
-        'link': (<Pool>addressesTags[i][1]).link,
-        'regexes': [],
-        'addresses': [addressesTags[i][0]],
-        'slug': ''
-      });
-    }
-
-    // Then, we find unique mining pool names
-    const poolNames: string[] = [];
-    for (let i = 0; i < poolsDuplicated.length; ++i) {
-      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
-        poolNames.push(poolsDuplicated[i].name);
+    for (const pool of this.miningPools) {
+      if (!pool.id) {
+        logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
+        continue;
       }
-    }
-    logger.debug(`Found ${poolNames.length} unique mining pools`);
 
-    // Get existing pools from the db
-    let existingPools;
-    try {
-      if (config.DATABASE.ENABLED === true) {
-        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
+      if (!poolDB) {
+        // New mining pool
+        const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.debug(`Inserting new mining pool ${pool.name}`);
+        await PoolsRepository.$insertNewMiningPool(pool, slug);
+        await this.$deleteUnknownBlocks();
       } else {
-        existingPools = [];
-      }
-    } catch (e) {
-      logger.err('Cannot get existing pools from the database, skipping pools.json import');
-      return;
-    }
-
-    this.miningPools = [];
-
-    // Finally, we generate the final consolidated pools data
-    const finalPoolDataAdd: Pool[] = [];
-    const finalPoolDataUpdate: Pool[] = [];
-    for (let i = 0; i < poolNames.length; ++i) {
-      let allAddresses: string[] = [];
-      let allRegexes: string[] = [];
-      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
-
-      for (let y = 0; y < match.length; ++y) {
-        allAddresses = allAddresses.concat(match[y].addresses);
-        allRegexes = allRegexes.concat(match[y].regexes);
-      }
-
-      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
-
-      let slug: string | undefined;
-      try {
-        slug = poolsJson['slugs'][poolNames[i]];
-      } catch (e) {
-        if (this.slugWarnFlag === false) {
-          logger.warn(`pools.json does not seem to contain the 'slugs' object`);
-          this.slugWarnFlag = true;
+        if (poolDB.name !== pool.name) {
+          // Pool has been renamed
+          const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+          logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
+          await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
+        }
+        if (poolDB.link !== pool.link) {
+          // Pool link has changed
+          logger.debug(`Updating link for ${pool.name} mining pool`);
+          await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
+        }
+        if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
+          JSON.stringify(pool.regexes) !== poolDB.regexes) {
+          // Pool addresses changed or coinbase tags changed
+          logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
+          await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
+          await this.$deleteBlocksForPool(poolDB);
         }
       }
-
-      if (slug === undefined) {
-        // Only keep alphanumerical
-        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
-      }
-
-      const poolObj = {
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': allRegexes,
-        'addresses': allAddresses,
-        'slug': slug
-      };
-
-      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
-      if (existingPool !== undefined) {
-        // Check if any data was actually updated
-        const equals = (a, b) =>
-          a.length === b.length &&
-          a.every((v, i) => v === b[i]);
-        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
-          finalPoolDataUpdate.push(poolObj);
-        }
-      } else {
-        logger.debug(`Add '${finalPoolName}' mining pool`);
-        finalPoolDataAdd.push(poolObj);
-      }
-
-      this.miningPools.push({
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': JSON.stringify(allRegexes),
-        'addresses': JSON.stringify(allAddresses),
-        'slug': slug
-      });
     }
 
-    if (config.DATABASE.ENABLED === false) { // Don't run db operations
-      logger.info('Mining pools.json import completed (no database)');
-      return;
-    }
-
-    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0) {    
-      logger.debug(`Update pools table now`);
-
-      // Add new mining pools into the database
-      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
-      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
-        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
-      }
-      queryAdd = queryAdd.slice(0, -1) + ';';
-
-      // Updated existing mining pools in the database
-      const updateQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
-        updateQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-          slug='${finalPoolDataUpdate[i].slug}'
-          WHERE name='${finalPoolDataUpdate[i].name}'
-        ;`);
-      }
-
-      try {
-        await this.$deleteBlocskToReindex(finalPoolDataUpdate);
-
-        if (finalPoolDataAdd.length > 0) {
-          await DB.query({ sql: queryAdd, timeout: 120000 });
-        }
-        for (const query of updateQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        await this.insertUnknownPool();
-        logger.info('Mining pools.json import completed');
-      } catch (e) {
-        logger.err(`Cannot import pools in the database`);
-        throw e;
-      }
-    }
-
-    try {
-      await this.insertUnknownPool();
-    } catch (e) {
-      logger.err(`Cannot insert unknown pool in the database`);
-      throw e;
-    }
+    logger.info('Mining pools-v2.json import completed');
   }
 
   /**
    * Manually add the 'unknown pool'
    */
-  private async insertUnknownPool() {
+  public async $insertUnknownPool(): Promise<void> {
+    if (!config.DATABASE.ENABLED) {
+      return;
+    }
+
     try {
       const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
       if (rows.length === 0) {
         await DB.query({
-          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
-          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
+          VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
         `});
       } else {
         await DB.query(`UPDATE pools
-          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
+          SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
           regexes='[]', addresses='[]',
-          slug='unknown'
-          WHERE name='Unknown'
+          slug='${this.unknownPool.slug}',
+          unique_id=0
+          WHERE slug='${this.unknownPool.slug}'
         `);
       }
     } catch (e) {
-      logger.err('Unable to insert "Unknown" mining pool');
+      logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
     }
   }
 
   /**
-   * Delete blocks which needs to be reindexed
+   * Delete indexed blocks for an updated mining pool
+   * 
+   * @param pool 
    */
-   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
     if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
       return;
     }
 
-    const blockCount = await BlocksRepository.$blockCount(null, null);
-    if (blockCount === 0) {
-      return;
-    }
-
-    for (const updatedPool of finalPoolDataUpdate) {
-      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
-      if (pool.length > 0) {
-        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`);
-        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
-      }
-    }
-
-    // Ignore early days of Bitcoin as there were not mining pool yet
-    logger.notice('Deleting blocks with unknown mining pool from height 130635 for future re-indexing');
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
+    // Ignore early days of Bitcoin as there were no mining pool yet
+    const [oldestPoolBlock]: any[] = await DB.query(`
+      SELECT height
+      FROM blocks
+      WHERE pool_id = ?
+      ORDER BY height
+      LIMIT 1`,
+      [pool.id]
+    );
+    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
     const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
+      [unknownPool[0].id]
+    );
+    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ?`,
+      [pool.id]
+    );
+  }
 
-    logger.notice('Truncating hashrates for future re-indexing');
-    await DB.query(`DELETE FROM hashrates`);
+  private async $deleteUnknownBlocks(): Promise<void> {
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= 130635`,
+      [unknownPool[0].id]
+    );
   }
 }
 

backend/src/api/rbf-cache.ts

@@ -1,31 +1,62 @@
-export interface CachedRbf {
-  txid: string;
-  expires: Date;
-}
+import { TransactionExtended } from "../mempool.interfaces";
 
 class RbfCache {
-  private cache: { [txid: string]: CachedRbf; } = {};
+  private replacedBy: { [txid: string]: string; } = {};
+  private replaces: { [txid: string]: string[] } = {};
+  private txs: { [txid: string]: TransactionExtended } = {};
+  private expiring: { [txid: string]: Date } = {};
 
   constructor() {
     setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
   }
 
-  public add(replacedTxId: string, newTxId: string): void {
-    this.cache[replacedTxId] = {
-      expires: new Date(Date.now() + 1000 * 604800), // 1 week
-      txid: newTxId,
-    };
+  public add(replacedTx: TransactionExtended, newTxId: string): void {
+    this.replacedBy[replacedTx.txid] = newTxId;
+    this.txs[replacedTx.txid] = replacedTx;
+    if (!this.replaces[newTxId]) {
+      this.replaces[newTxId] = [];
+    }
+    this.replaces[newTxId].push(replacedTx.txid);
   }
 
-  public get(txId: string): CachedRbf | undefined {
-    return this.cache[txId];
+  public getReplacedBy(txId: string): string | undefined {
+    return this.replacedBy[txId];
+  }
+
+  public getReplaces(txId: string): string[] | undefined {
+    return this.replaces[txId];
+  }
+
+  public getTx(txId: string): TransactionExtended | undefined {
+    return this.txs[txId];
+  }
+
+  // flag a transaction as removed from the mempool
+  public evict(txid): void {
+    this.expiring[txid] = new Date(Date.now() + 1000 * 86400); // 24 hours
   }
 
   private cleanup(): void {
     const currentDate = new Date();
-    for (const c in this.cache) {
-      if (this.cache[c].expires < currentDate) {
-        delete this.cache[c];
+    for (const txid in this.expiring) {
+      if (this.expiring[txid] < currentDate) {
+        delete this.expiring[txid];
+        this.remove(txid);
+      }
+    }
+  }
+
+  // remove a transaction & all previous versions from the cache
+  private remove(txid): void {
+    // don't remove a transaction while a newer version remains in the mempool
+    if (this.replaces[txid] && !this.replacedBy[txid]) {
+      const replaces = this.replaces[txid];
+      delete this.replaces[txid];
+      for (const tx of replaces) {
+        // recursively remove prior versions from the cache
+        delete this.replacedBy[tx];
+        delete this.txs[tx];
+        this.remove(tx);
       }
     }
   }

backend/src/api/statistics/statistics-api.ts

@@ -375,6 +375,17 @@ class StatisticsApi {
     }
   }
 
+  public async $list4Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(43200, '4 YEAR'); // 12h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list4Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
   private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
     return statistic.map((s) => {
       return {

backend/src/api/statistics/statistics.routes.ts

@@ -14,10 +14,11 @@ class StatisticsRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', this.$getStatisticsByTime.bind(this, '1y'))
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y'))
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/4y', this.$getStatisticsByTime.bind(this, '4y'))
     ;
   }
 
-  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y', req: Request, res: Response) {
+  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y', req: Request, res: Response) {
     res.header('Pragma', 'public');
     res.header('Cache-control', 'public');
     res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
@@ -54,6 +55,9 @@ class StatisticsRoutes {
         case '3y':
           result = await statisticsApi.$list3Y();
           break;
+        case '4y':
+          result = await statisticsApi.$list4Y();
+          break;
         default:
           result = await statisticsApi.$list2H();
       }

backend/src/api/transaction-utils.ts

@@ -1,8 +1,7 @@
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
-import config from '../config';
 import { Common } from './common';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 
 class TransactionUtils {
   constructor() { }
@@ -15,14 +14,26 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)
     };
   }
 
-  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
-    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+  /**
+   * @param txId 
+   * @param addPrevouts 
+   * @param lazyPrevouts 
+   * @param forceCore - See https://github.com/mempool/mempool/issues/2904
+   */
+  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<TransactionExtended> {
+    let transaction: IEsploraApi.Transaction;
+    if (forceCore === true) {
+      transaction  = await bitcoinCoreApi.$getRawTransaction(txId, true);
+    } else {
+      transaction  = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+    }
     return this.extendTransaction(transaction);
   }
 

backend/src/api/tx-selection-worker.ts

@@ -0,0 +1,294 @@
+import config from '../config';
+import logger from '../logger';
+import { ThreadTransaction, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces';
+import { PairingHeap } from '../utils/pairing-heap';
+import { Common } from './common';
+import { parentPort } from 'worker_threads';
+
+let mempool: { [txid: string]: ThreadTransaction } = {};
+
+if (parentPort) {
+  parentPort.on('message', (params) => {
+    if (params.type === 'set') {
+      mempool = params.mempool;
+    } else if (params.type === 'update') {
+      params.added.forEach(tx => {
+        mempool[tx.txid] = tx;
+      });
+      params.removed.forEach(txid => {
+        delete mempool[txid];
+      });
+    }
+    
+    const { blocks, clusters } = makeBlockTemplates(mempool);
+
+    // return the result to main thread.
+    if (parentPort) {
+     parentPort.postMessage({ blocks, clusters });
+    }
+  });
+}
+
+/*
+* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+*/
+function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
+  : { blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } } {
+  const start = Date.now();
+  const auditPool: { [txid: string]: AuditTransaction } = {};
+  const mempoolArray: AuditTransaction[] = [];
+  const restOfArray: ThreadTransaction[] = [];
+  const cpfpClusters: { [root: string]: string[] } = {};
+  
+  // grab the top feerate txs up to maxWeight
+  Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+    // initializing everything up front helps V8 optimize property access later
+    auditPool[tx.txid] = {
+      txid: tx.txid,
+      fee: tx.fee,
+      weight: tx.weight,
+      feePerVsize: tx.feePerVsize,
+      effectiveFeePerVsize: tx.feePerVsize,
+      vin: tx.vin,
+      relativesSet: false,
+      ancestorMap: new Map<string, AuditTransaction>(),
+      children: new Set<AuditTransaction>(),
+      ancestorFee: 0,
+      ancestorWeight: 0,
+      score: 0,
+      used: false,
+      modified: false,
+      modifiedNode: null,
+    };
+    mempoolArray.push(auditPool[tx.txid]);
+  });
+
+  // Build relatives graph & calculate ancestor scores
+  for (const tx of mempoolArray) {
+    if (!tx.relativesSet) {
+      setRelatives(tx, auditPool);
+    }
+  }
+
+  // Sort by descending ancestor score
+  mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+
+  // Build blocks by greedily choosing the highest feerate package
+  // (i.e. the package rooted in the transaction with the best ancestor score)
+  const blocks: ThreadTransaction[][] = [];
+  let blockWeight = 4000;
+  let blockSize = 0;
+  let transactions: AuditTransaction[] = [];
+  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+  let overflow: AuditTransaction[] = [];
+  let failures = 0;
+  let top = 0;
+  while ((top < mempoolArray.length || !modified.isEmpty())) {
+    // skip invalid transactions
+    while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
+      top++;
+    }
+
+    // Select best next package
+    let nextTx;
+    const nextPoolTx = mempoolArray[top];
+    const nextModifiedTx = modified.peek();
+    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+      nextTx = nextPoolTx;
+      top++;
+    } else {
+      modified.pop();
+      if (nextModifiedTx) {
+        nextTx = nextModifiedTx;
+        nextTx.modifiedNode = undefined;
+      }
+    }
+
+    if (nextTx && !nextTx?.used) {
+      // Check if the package fits into this block
+      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+        const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
+        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+        let isCluster = false;
+        if (sortedTxSet.length > 1) {
+          cpfpClusters[nextTx.txid] = sortedTxSet.map(tx => tx.txid);
+          isCluster = true;
+        }
+        const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
+        const used: AuditTransaction[] = [];
+        while (sortedTxSet.length) {
+          const ancestor = sortedTxSet.pop();
+          const mempoolTx = mempool[ancestor.txid];
+          ancestor.used = true;
+          ancestor.usedBy = nextTx.txid;
+          // update original copy of this tx with effective fee rate & relatives data
+          mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+          if (isCluster) {
+            mempoolTx.cpfpRoot = nextTx.txid;
+          }
+          mempoolTx.cpfpChecked = true;
+          transactions.push(ancestor);
+          blockSize += ancestor.size;
+          blockWeight += ancestor.weight;
+          used.push(ancestor);
+        }
+
+        // remove these as valid package ancestors for any descendants remaining in the mempool
+        if (used.length) {
+          used.forEach(tx => {
+            updateDescendants(tx, auditPool, modified);
+          });
+        }
+
+        failures = 0;
+      } else {
+        // hold this package in an overflow list while we check for smaller options
+        overflow.push(nextTx);
+        failures++;
+      }
+    }
+
+    // this block is full
+    const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+    const queueEmpty = top >= mempoolArray.length && modified.isEmpty();
+    if ((exceededPackageTries || queueEmpty) && blocks.length < 7) {
+      // construct this block
+      if (transactions.length) {
+        blocks.push(transactions.map(t => mempool[t.txid]));
+      }
+      // reset for the next block
+      transactions = [];
+      blockSize = 0;
+      blockWeight = 4000;
+
+      // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+      for (const overflowTx of overflow.reverse()) {
+        if (overflowTx.modified) {
+          overflowTx.modifiedNode = modified.add(overflowTx);
+        } else {
+          top--;
+          mempoolArray[top] = overflowTx;
+        }
+      }
+      overflow = [];
+    }
+  }
+  // pack any leftover transactions into the last block
+  for (const tx of overflow) {
+    if (!tx || tx?.used) {
+      continue;
+    }
+    blockWeight += tx.weight;
+    const mempoolTx = mempool[tx.txid];
+    // update original copy of this tx with effective fee rate & relatives data
+    mempoolTx.effectiveFeePerVsize = tx.score;
+    if (tx.ancestorMap.size > 0) {
+      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
+      mempoolTx.cpfpRoot = tx.txid;
+    }
+    mempoolTx.cpfpChecked = true;
+    transactions.push(tx);
+    tx.used = true;
+  }
+  const blockTransactions = transactions.map(t => mempool[t.txid]);
+  restOfArray.forEach(tx => {
+    blockWeight += tx.weight;
+    tx.effectiveFeePerVsize = tx.feePerVsize;
+    tx.cpfpChecked = false;
+    blockTransactions.push(tx);
+  });
+  if (blockTransactions.length) {
+    blocks.push(blockTransactions);
+  }
+  transactions = [];
+
+  const end = Date.now();
+  const time = end - start;
+  logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+  return { blocks, clusters: cpfpClusters };
+}
+
+// traverse in-mempool ancestors
+// recursion unavoidable, but should be limited to depth < 25 by mempool policy
+function setRelatives(
+  tx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+): void {
+  for (const parent of tx.vin) {
+    const parentTx = mempool[parent];
+    if (parentTx && !tx.ancestorMap?.has(parent)) {
+      tx.ancestorMap.set(parent, parentTx);
+      parentTx.children.add(tx);
+      // visit each node only once
+      if (!parentTx.relativesSet) {
+        setRelatives(parentTx, mempool);
+      }
+      parentTx.ancestorMap.forEach((ancestor) => {
+        tx.ancestorMap.set(ancestor.txid, ancestor);
+      });
+    }
+  };
+  tx.ancestorFee = tx.fee || 0;
+  tx.ancestorWeight = tx.weight || 0;
+  tx.ancestorMap.forEach((ancestor) => {
+    tx.ancestorFee += ancestor.fee;
+    tx.ancestorWeight += ancestor.weight;
+  });
+  tx.score = tx.ancestorFee / ((tx.ancestorWeight / 4) || 1);
+  tx.relativesSet = true;
+}
+
+// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+// avoids recursion to limit call stack depth
+function updateDescendants(
+  rootTx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+  modified: PairingHeap<AuditTransaction>,
+): void {
+  const descendantSet: Set<AuditTransaction> = new Set();
+  // stack of nodes left to visit
+  const descendants: AuditTransaction[] = [];
+  let descendantTx;
+  let tmpScore;
+  rootTx.children.forEach(childTx => {
+    if (!descendantSet.has(childTx)) {
+      descendants.push(childTx);
+      descendantSet.add(childTx);
+    }
+  });
+  while (descendants.length) {
+    descendantTx = descendants.pop();
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+      // remove tx as ancestor
+      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorFee -= rootTx.fee;
+      descendantTx.ancestorWeight -= rootTx.weight;
+      tmpScore = descendantTx.score;
+      descendantTx.score = descendantTx.ancestorFee / (descendantTx.ancestorWeight / 4);
+
+      if (!descendantTx.modifiedNode) {
+        descendantTx.modified = true;
+        descendantTx.modifiedNode = modified.add(descendantTx);
+      } else {
+        // rebalance modified heap if score has changed
+        if (descendantTx.score < tmpScore) {
+          modified.decreasePriority(descendantTx.modifiedNode);
+        } else if (descendantTx.score > tmpScore) {
+          modified.increasePriority(descendantTx.modifiedNode);
+        }
+      }
+
+      // add this node's children to the stack
+      descendantTx.children.forEach(childTx => {
+        // visit each node only once
+        if (!descendantSet.has(childTx)) {
+          descendants.push(childTx);
+          descendantSet.add(childTx);
+        }
+      });
+    }
+  }
+}

backend/src/api/websocket-handler.ts

@@ -1,14 +1,13 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
 import {
-  BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
-  OptimizedStatistic, ILoadingIndicators, IConversionRates
+  BlockExtended, TransactionExtended, WebsocketResponse,
+  OptimizedStatistic, ILoadingIndicators
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
 import mempoolBlocks from './mempool-blocks';
-import fiatConversion from './fiat-conversion';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import config from '../config';
@@ -18,6 +17,10 @@ import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import Audit from './audit';
+import { deepClone } from '../utils/clone';
+import priceUpdater from '../tasks/price-updater';
+import { ApiPrice } from '../repositories/PricesRepository';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -57,10 +60,10 @@ class WebsocketHandler {
               client['track-tx'] = parsedMessage['track-tx'];
               // Client is telling the transaction wasn't found
               if (parsedMessage['watch-mempool']) {
-                const rbfCacheTx = rbfCache.get(client['track-tx']);
-                if (rbfCacheTx) {
+                const rbfCacheTxid = rbfCache.getReplacedBy(client['track-tx']);
+                if (rbfCacheTxid) {
                   response['txReplaced'] = {
-                    txid: rbfCacheTx.txid,
+                    txid: rbfCacheTxid,
                   };
                   client['track-tx'] = null;
                 } else {
@@ -191,7 +194,7 @@ class WebsocketHandler {
     });
   }
 
-  handleNewConversionRates(conversionRates: IConversionRates) {
+  handleNewConversionRates(conversionRates: ApiPrice) {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
@@ -212,7 +215,7 @@ class WebsocketHandler {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
       'blocks': _blocks,
-      'conversions': fiatConversion.getConversionRates(),
+      'conversions': priceUpdater.getLatestPrices(),
       'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
@@ -243,13 +246,18 @@ class WebsocketHandler {
     });
   }
 
-  handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
-    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) {
+  async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    mempoolBlocks.updateMempoolBlocks(newMempool);
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(newMempool, true);
+    }
+
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     const mempoolInfo = memPool.getMempoolInfo();
@@ -404,54 +412,39 @@ class WebsocketHandler {
       }
     });
   }
-
-  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
+ 
+  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    let mBlocks: undefined | MempoolBlock[];
-    let mBlockDeltas: undefined | MempoolBlockDelta[];
-    let matchRate = 0;
     const _memPool = memPool.getMempool();
-    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
 
-    if (_mempoolBlocks[0]) {
-      const matches: string[] = [];
-      const added: string[] = [];
-      const missing: string[] = [];
-
-      for (const txId of txIds) {
-        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
-          matches.push(txId);
-        } else {
-          added.push(txId);
-        }
-        delete _memPool[txId];
+    if (config.MEMPOOL.AUDIT) {
+      let projectedBlocks;
+      // template calculation functions have mempool side effects, so calculate audits using
+      // a cloned copy of the mempool if we're running a different algorithm for mempool updates
+      const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
+      if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+        projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
+      } else {
+        projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
       }
 
-      for (const txId of _mempoolBlocks[0].transactionIds) {
-        if (matches.includes(txId) || added.includes(txId)) {
-          continue;
-        }
-        missing.push(txId);
-      }
+      if (Common.indexingEnabled() && memPool.isInSync()) {
+        const { censored, added, fresh, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+        const matchRate = Math.round(score * 100 * 100) / 100;
 
-      matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
-      mempoolBlocks.updateMempoolBlocks(_memPool);
-      mBlocks = mempoolBlocks.getMempoolBlocks();
-      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
-
-      if (Common.indexingEnabled()) {
-        const stripped = _mempoolBlocks[0].transactions.map((tx) => {
+        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
           return {
             txid: tx.txid,
             vsize: tx.vsize,
             fee: tx.fee ? Math.round(tx.fee) : 0,
             value: tx.value,
           };
-        });  
-        BlocksSummariesRepository.$saveSummary({
+        }) : [];
+
+        BlocksSummariesRepository.$saveTemplate({
           height: block.height,
           template: {
             id: block.id,
@@ -464,16 +457,39 @@ class WebsocketHandler {
           height: block.height,
           hash: block.id,
           addedTxs: added,
-          missingTxs: missing,
+          missingTxs: censored,
+          freshTxs: fresh,
           matchRate: matchRate,
         });
+
+        if (block.extras) {
+          block.extras.matchRate = matchRate;
+          block.extras.similarity = similarity;
+        }
+      }
+    } else if (block.extras) {
+      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+      if (mBlocks?.length && mBlocks[0].transactions) {
+        block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
       }
     }
 
-    if (block.extras) {
-      block.extras.matchRate = matchRate;
+    const removed: string[] = [];
+    // Update mempool to remove transactions included in the new block
+    for (const txId of txIds) {
+      delete _memPool[txId];
+      removed.push(txId);
+      rbfCache.evict(txId);
     }
 
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(_memPool, true);
+    }
+    const mBlocks = mempoolBlocks.getMempoolBlocks();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
     const da = difficultyAdjustment.getDifficultyAdjustment();
     const fees = feeApi.getRecommendedFee();
 

backend/src/config.ts

@@ -1,7 +1,10 @@
-const configFile = require('../mempool-config.json');
+const configFromFile = require(
+    process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
+);
 
 interface IConfig {
   MEMPOOL: {
+    ENABLED: boolean;
     NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
@@ -16,7 +19,6 @@ interface IConfig {
     MEMPOOL_BLOCKS_AMOUNT: number;
     INDEXING_BLOCKS_AMOUNT: number;
     BLOCKS_SUMMARIES_INDEXING: boolean;
-    PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
     EXTERNAL_MAX_RETRY: number;
@@ -24,6 +26,14 @@ interface IConfig {
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
     AUTOMATIC_BLOCK_REINDEXING: boolean;
+    POOLS_JSON_URL: string,
+    POOLS_JSON_TREE_URL: string,
+    AUDIT: boolean;
+    ADVANCED_GBT_AUDIT: boolean;
+    ADVANCED_GBT_MEMPOOL: boolean;
+    CPFP_INDEXING: boolean;
+    MAX_BLOCKS_BULK_QUERY: number;
+    DISK_CACHE_BLOCK_INTERVAL: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -31,10 +41,19 @@ interface IConfig {
   LIGHTNING: {
     ENABLED: boolean;
     BACKEND: 'lnd' | 'cln' | 'ldk';
+    TOPOLOGY_FOLDER: string;
+    STATS_REFRESH_INTERVAL: number;
+    GRAPH_REFRESH_INTERVAL: number;
+    LOGGER_UPDATE_INTERVAL: number;
+    FORENSICS_INTERVAL: number;
+    FORENSICS_RATE_LIMIT: number;
   };
   LND: {
     TLS_CERT_PATH: string;
     MACAROON_PATH: string;
+    REST_API_URL: string;
+  };
+  CLIGHTNING: {
     SOCKET: string;
   };
   ELECTRUM: {
@@ -108,6 +127,7 @@ interface IConfig {
 
 const defaults: IConfig = {
   'MEMPOOL': {
+    'ENABLED': true,
     'NETWORK': 'mainnet',
     'BACKEND': 'none',
     'HTTP_PORT': 8999,
@@ -122,7 +142,6 @@ const defaults: IConfig = {
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
     'BLOCKS_SUMMARIES_INDEXING': false,
-    'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [],
     'EXTERNAL_MAX_RETRY': 1,
@@ -130,6 +149,14 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+    'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'AUDIT': false,
+    'ADVANCED_GBT_AUDIT': false,
+    'ADVANCED_GBT_MEMPOOL': false,
+    'CPFP_INDEXING': false,
+    'MAX_BLOCKS_BULK_QUERY': 0,
+    'DISK_CACHE_BLOCK_INTERVAL': 6,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -177,12 +204,21 @@ const defaults: IConfig = {
   },
   'LIGHTNING': {
     'ENABLED': false,
-    'BACKEND': 'lnd'
+    'BACKEND': 'lnd',
+    'TOPOLOGY_FOLDER': '',
+    'STATS_REFRESH_INTERVAL': 600,
+    'GRAPH_REFRESH_INTERVAL': 600,
+    'LOGGER_UPDATE_INTERVAL': 30,
+    'FORENSICS_INTERVAL': 43200,
+    'FORENSICS_RATE_LIMIT': 20,
   },
   'LND': {
     'TLS_CERT_PATH': '',
     'MACAROON_PATH': '',
-    'SOCKET': 'localhost:10009',
+    'REST_API_URL': 'https://localhost:8080',
+  },
+  'CLIGHTNING': {
+    'SOCKET': '',
   },
   'SOCKS5PROXY': {
     'ENABLED': false,
@@ -204,11 +240,11 @@ const defaults: IConfig = {
     'BISQ_URL': 'https://bisq.markets/api',
     'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
   },
-  "MAXMIND": {
+  'MAXMIND': {
     'ENABLED': false,
-    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
-    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
+    'GEOLITE2_CITY': '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
+    'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
+    'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
   },
 };
 
@@ -224,13 +260,14 @@ class Config implements IConfig {
   BISQ: IConfig['BISQ'];
   LIGHTNING: IConfig['LIGHTNING'];
   LND: IConfig['LND'];
+  CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
 
   constructor() {
-    const configs = this.merge(configFile, defaults);
+    const configs = this.merge(configFromFile, defaults);
     this.MEMPOOL = configs.MEMPOOL;
     this.ESPLORA = configs.ESPLORA;
     this.ELECTRUM = configs.ELECTRUM;
@@ -242,6 +279,7 @@ class Config implements IConfig {
     this.BISQ = configs.BISQ;
     this.LIGHTNING = configs.LIGHTNING;
     this.LND = configs.LND;
+    this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;

backend/src/database.ts

@@ -1,7 +1,7 @@
 import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
-import { PoolOptions } from 'mysql2/typings/mysql';
+import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
 
  class DB {
   constructor() {
@@ -24,11 +24,14 @@ import { PoolOptions } from 'mysql2/typings/mysql';
 
   private checkDBFlag() {
     if (config.DATABASE.ENABLED === false) {
-      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+      const stack = new Error().stack;
+      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
     }
   }
 
-  public async query(query, params?) {
+  public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
+    OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
+  {
     this.checkDBFlag();
     const pool = await this.getPool();
     return pool.query(query, params);

backend/src/index.ts

@@ -1,4 +1,4 @@
-import express from "express";
+import express from 'express';
 import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
@@ -10,7 +10,6 @@ import memPool from './api/mempool';
 import diskCache from './api/disk-cache';
 import statistics from './api/statistics/statistics';
 import websocketHandler from './api/websocket-handler';
-import fiatConversion from './api/fiat-conversion';
 import bisq from './api/bisq/bisq';
 import bisqMarkets from './api/bisq/markets';
 import logger from './logger';
@@ -28,12 +27,19 @@ import nodesRoutes from './api/explorer/nodes.routes';
 import channelsRoutes from './api/explorer/channels.routes';
 import generalLightningRoutes from './api/explorer/general.routes';
 import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
-import nodeSyncService from './tasks/lightning/node-sync.service';
-import statisticsRoutes from "./api/statistics/statistics.routes";
-import miningRoutes from "./api/mining/mining-routes";
-import bisqRoutes from "./api/bisq/bisq.routes";
-import liquidRoutes from "./api/liquid/liquid.routes";
-import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
+import networkSyncService from './tasks/lightning/network-sync.service';
+import statisticsRoutes from './api/statistics/statistics.routes';
+import miningRoutes from './api/mining/mining-routes';
+import bisqRoutes from './api/bisq/bisq.routes';
+import liquidRoutes from './api/liquid/liquid.routes';
+import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
+import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
+import forensicsService from './tasks/lightning/forensics.service';
+import priceUpdater from './tasks/price-updater';
+import chainTips from './api/chain-tips';
+import { AxiosError } from 'axios';
+import v8 from 'v8';
+import { formatBytes, getBytesUnit } from './utils/format';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -41,6 +47,11 @@ class Server {
   private app: Application;
   private currentBackendRetryInterval = 5;
 
+  private maxHeapSize: number = 0;
+  private heapLogInterval: number = 60;
+  private warnedHeapCritical: boolean = false;
+  private lastHeapLogTime: number | null = null;
+
   constructor() {
     this.app = express();
 
@@ -73,42 +84,43 @@ class Server {
     }
   }
 
-  async startServer(worker = false) {
+  async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
+    if (config.DATABASE.ENABLED) {
+      await DB.checkDbConnection();
+      try {
+        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
+          await databaseMigration.$blocksReindexingTruncate();
+        }
+        await databaseMigration.$initializeOrMigrateDatabase();
+      } catch (e) {
+        throw new Error(e instanceof Error ? e.message : 'Error');
+      }
+    }
+
     this.app
       .use((req: Request, res: Response, next: NextFunction) => {
         res.setHeader('Access-Control-Allow-Origin', '*');
         next();
       })
       .use(express.urlencoded({ extended: true }))
-      .use(express.text())
+      .use(express.text({ type: ['text/plain', 'application/base64'] }))
       ;
 
+    if (config.DATABASE.ENABLED) {
+      await priceUpdater.$initializeLatestPriceWithDb();
+    }
+
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
 
     this.setUpWebsocketHandling();
 
+    await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
-    diskCache.loadMempoolCache();
-
-    if (config.DATABASE.ENABLED) {
-      await DB.checkDbConnection();
-      try {
-        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
-          const tables = process.env.npm_config_reindex.split(',');
-          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
-          await Common.sleep$(5000);
-          await databaseMigration.$truncateIndexedData(tables);
-        }
-        await databaseMigration.$initializeOrMigrateDatabase();
-        if (Common.indexingEnabled()) {
-          await indexer.$resetHashratesIndexingState();
-        }
-      } catch (e) {
-        throw new Error(e instanceof Error ? e.message : 'Error');
-      }
+    if (config.MEMPOOL.ENABLED) {
+      diskCache.loadMempoolCache();
     }
 
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
@@ -123,10 +135,16 @@ class Server {
       }
     }
 
-    fiatConversion.startService();
+    priceUpdater.$run();
+    await chainTips.updateOrphanedBlocks();
 
     this.setUpHttpApiRoutes();
-    this.runMainUpdateLoop();
+
+    if (config.MEMPOOL.ENABLED) {
+      this.runMainUpdateLoop();
+    }
+
+    setInterval(() => { this.healthCheck(); }, 2500);
 
     if (config.BISQ.ENABLED) {
       bisq.startBisqService();
@@ -136,8 +154,7 @@ class Server {
     }
 
     if (config.LIGHTNING.ENABLED) {
-      nodeSyncService.$startService()
-        .then(() => lightningStatsUpdater.$startService());
+      this.$runLightningBackend();
     }
 
     this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
@@ -149,7 +166,7 @@ class Server {
     });
   }
 
-  async runMainUpdateLoop() {
+  async runMainUpdateLoop(): Promise<void> {
     try {
       try {
         await memPool.$updateMemPoolInfo();
@@ -161,29 +178,50 @@ class Server {
           logger.debug(msg);
         }
       }
-      await poolsUpdater.updatePoolsJson();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
-    } catch (e) {
-      const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
+    } catch (e: any) {
+      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
+      loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
+      if (e?.stack) {
+        loggerMsg += ` Stack trace: ${e.stack}`;
+      }
+      // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
+      // From the second Exception, `logger.warn` the Exception and increase the retry delay
+      // Maximum retry delay is 60 seconds
       if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
         logger.debug(loggerMsg);
       }
-      logger.debug(JSON.stringify(e));
+      if (e instanceof AxiosError) {
+        logger.debug(`AxiosError: ${e?.message}`);
+      }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
       this.currentBackendRetryInterval *= 2;
       this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
     }
   }
 
-  setUpWebsocketHandling() {
+  async $runLightningBackend(): Promise<void> {
+    try {
+      await fundingTxFetcher.$init();
+      await networkSyncService.$startService();
+      await lightningStatsUpdater.$startService();
+      await forensicsService.$startService();
+    } catch(e) {
+      logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      await Common.sleep$(1000 * 60);
+      this.$runLightningBackend();
+    };
+}
+
+  setUpWebsocketHandling(): void {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);
     }
@@ -197,19 +235,21 @@ class Server {
       });
     }
     websocketHandler.setupConnectionHandling();
-    statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
-    blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
-    memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
-    fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
+    if (config.MEMPOOL.ENABLED) {
+      statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
+      memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+    }
+    priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
-
-  setUpHttpApiRoutes() {
+  
+  setUpHttpApiRoutes(): void {
     bitcoinRoutes.initRoutes(this.app);
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
       statisticsRoutes.initRoutes(this.app);
     }
-    if (Common.indexingEnabled()) {
+    if (Common.indexingEnabled() && config.MEMPOOL.ENABLED) {
       miningRoutes.initRoutes(this.app);
     }
     if (config.BISQ.ENABLED) {
@@ -224,6 +264,26 @@ class Server {
       channelsRoutes.initRoutes(this.app);
     }
   }
+
+  healthCheck(): void {
+    const now = Date.now();
+    const stats = v8.getHeapStatistics();
+    this.maxHeapSize = Math.max(stats.used_heap_size, this.maxHeapSize);
+    const warnThreshold = 0.8 * stats.heap_size_limit;
+
+    const byteUnits = getBytesUnit(Math.max(this.maxHeapSize, stats.heap_size_limit));
+
+    if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
+      this.warnedHeapCritical = true;
+      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
+    }
+    if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
+      logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);
+      this.warnedHeapCritical = false;
+      this.maxHeapSize = 0;
+      this.lastHeapLogTime = now;
+    }
+  }
 }
 
-const server = new Server();
+((): Server => new Server())();

backend/src/indexer.ts

@@ -3,24 +3,100 @@ import blocks from './api/blocks';
 import mempool from './api/mempool';
 import mining from './api/mining/mining';
 import logger from './logger';
-import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
+import PricesRepository from './repositories/PricesRepository';
+
+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
 
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
+  tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];
 
-  constructor() {
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
   }
 
-  public reindex() {
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
     if (Common.indexingEnabled()) {
       this.runIndexer = true;
     }
   }
 
-  public async $run() {
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
+    if (!Common.indexingEnabled()) {
+      return;
+    }
+
+    if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      const lastestPriceId = await PricesRepository.$getLatestPriceId();
+      if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
+        setTimeout(() => {
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+          this.runSingleTask('blocksPrices');
+        }, 10000);
+      } else {
+        logger.debug(`Blocks prices indexer will run now`, logger.tags.mining);
+        await mining.$indexBlockPrices();
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+      }
+    }
+
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
+  }
+
+  public async $run(): Promise<void> {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
     ) {
@@ -38,24 +114,26 @@ class Indexer {
 
     logger.debug(`Running mining indexer`);
 
+    await this.checkAvailableCoreIndexes();
+
     try {
       await priceUpdater.$run();
 
       const chainValid = await blocks.$generateBlockDatabase();
       if (chainValid === false) {
         // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
-        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining);
         setTimeout(() => this.reindex(), 10000);
         this.indexerRunning = false;
         return;
       }
 
-      await mining.$indexBlockPrices();
+      this.runSingleTask('blocksPrices');
       await mining.$indexDifficultyAdjustments();
-      await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();
       await blocks.$generateBlocksSummariesDatabase();
+      await blocks.$generateCPFPDatabase();
     } catch (e) {
       this.indexerRunning = false;
       logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
@@ -70,16 +148,6 @@ class Indexer {
     logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
     setTimeout(() => this.reindex(), runEvery);
   }
-
-  async $resetHashratesIndexingState() {
-    try {
-      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
-      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
-    } catch (e) {
-      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
 }
 
 export default new Indexer();

backend/src/logger.ts

@@ -32,22 +32,27 @@ class Logger {
     local7: 23
   };
 
+  public tags = {
+    mining: 'Mining',
+    ln: 'Lightning',
+  };  
+
   // @ts-ignore
-  public emerg: ((msg: string) => void);
+  public emerg: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public alert: ((msg: string) => void);
+  public alert: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public crit: ((msg: string) => void);
+  public crit: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public err: ((msg: string) => void);
+  public err: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public warn: ((msg: string) => void);
+  public warn: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public notice: ((msg: string) => void);
+  public notice: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public info: ((msg: string) => void);
+  public info: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public debug: ((msg: string) => void);
+  public debug: ((msg: string, tag?: string) => void);
 
   private name = 'mempool';
   private client: dgram.Socket;
@@ -66,15 +71,15 @@ class Logger {
 
   private addprio(prio): void {
     this[prio] = (function(_this) {
-      return function(msg) {
-        return _this.msg(prio, msg);
+      return function(msg, tag?: string) {
+        return _this.msg(prio, msg, tag);
       };
     })(this);
   }
 
   private getNetwork(): string {
     if (config.LIGHTNING.ENABLED) {
-      return 'lightning';
+      return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; 
     }
     if (config.BISQ.ENABLED) {
       return 'bisq';
@@ -85,7 +90,7 @@ class Logger {
     return '';
   }
 
-  private msg(priority, msg) {
+  private msg(priority, msg, tag?: string) {
     let consolemsg, prionum, syslogmsg;
     if (typeof msg === 'string' && msg.length > 0) {
       while (msg[msg.length - 1].charCodeAt(0) === 10) {
@@ -94,10 +99,10 @@ class Logger {
     }
     const network = this.network ? ' <' + this.network + '>' : '';
     prionum = Logger.priorities[priority] || Logger.priorities.info;
-    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${msg}`;
+    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
 
     if (config.SYSLOG.ENABLED && Logger.priorities[priority] <= Logger.priorities[config.SYSLOG.MIN_PRIORITY]) {
-      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
+      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
       this.syslog(syslogmsg);
     }
     if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {

backend/src/mempool.interfaces.ts

@@ -1,7 +1,10 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { OrphanedBlock } from './api/chain-tips';
+import { HeapNode } from './utils/pairing-heap';
 
 export interface PoolTag {
-  id: number; // mysql row id
+  id: number;
+  uniqueId: number;
   name: string;
   link: string;
   regexes: string; // JSON array
@@ -15,6 +18,7 @@ export interface PoolInfo {
   link: string;
   blockCount: number;
   slug: string;
+  avgMatchRate: number | null;
 }
 
 export interface PoolStats extends PoolInfo {
@@ -27,10 +31,16 @@ export interface BlockAudit {
   height: number,
   hash: string,
   missingTxs: string[],
+  freshTxs: string[],
   addedTxs: string[],
   matchRate: number,
 }
 
+export interface AuditScore {
+  hash: string,
+  matchRate?: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;
@@ -56,6 +66,7 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -65,17 +76,57 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   firstSeen?: number;
   effectiveFeePerVsize: number;
   ancestors?: Ancestor[];
+  descendants?: Ancestor[];
   bestDescendant?: BestDescendant | null;
   cpfpChecked?: boolean;
   deleteAfter?: number;
 }
 
-interface Ancestor {
+export interface AuditTransaction {
+  txid: string;
+  fee: number;
+  weight: number;
+  feePerVsize: number;
+  effectiveFeePerVsize: number;
+  vin: string[];
+  relativesSet: boolean;
+  ancestorMap: Map<string, AuditTransaction>;
+  children: Set<AuditTransaction>;
+  ancestorFee: number;
+  ancestorWeight: number;
+  score: number;
+  used: boolean;
+  modified: boolean;
+  modifiedNode: HeapNode<AuditTransaction>;
+}
+
+export interface ThreadTransaction {
+  txid: string;
+  fee: number;
+  weight: number;
+  feePerVsize: number;
+  effectiveFeePerVsize?: number;
+  vin: string[];
+  cpfpRoot?: string;
+  cpfpChecked?: boolean;
+}
+
+export interface Ancestor {
   txid: string;
   weight: number;
   fee: number;
 }
 
+export interface TransactionSet {
+  fee: number;
+  weight: number;
+  score: number;
+  children?: Set<string>;
+  available?: boolean;
+  modified?: boolean;
+  modifiedNode?: HeapNode<string>;
+}
+
 interface BestDescendant {
   txid: string;
   weight: number;
@@ -84,7 +135,9 @@ interface BestDescendant {
 
 export interface CpfpInfo {
   ancestors: Ancestor[];
-  bestDescendant: BestDescendant | null;
+  bestDescendant?: BestDescendant | null;
+  descendants?: Ancestor[];
+  effectiveFeePerVsize?: number;
 }
 
 export interface TransactionStripped {
@@ -95,23 +148,45 @@ export interface TransactionStripped {
 }
 
 export interface BlockExtension {
-  totalFees?: number;
-  medianFee?: number;
-  feeRange?: number[];
-  reward?: number;
-  coinbaseTx?: TransactionMinerInfo;
-  matchRate?: number;
-  pool?: {
-    id: number;
+  totalFees: number;
+  medianFee: number; // median fee rate
+  feeRange: number[]; // fee rate percentiles
+  reward: number;
+  matchRate: number | null;
+  similarity?: number;
+  pool: {
+    id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
     name: string;
     slug: string;
   };
-  avgFee?: number;
-  avgFeeRate?: number;
-  coinbaseRaw?: string;
-  usd?: number | null;
+  avgFee: number;
+  avgFeeRate: number;
+  coinbaseRaw: string;
+  orphans: OrphanedBlock[] | null;
+  coinbaseAddress: string | null;
+  coinbaseSignature: string | null;
+  coinbaseSignatureAscii: string | null;
+  virtualSize: number;
+  avgTxSize: number;
+  totalInputs: number;
+  totalOutputs: number;
+  totalOutputAmt: number;
+  medianFeeAmt: number | null; // median fee in sats
+  feePercentiles: number[] | null, // fee percentiles in sats
+  segwitTotalTxs: number;
+  segwitTotalSize: number;
+  segwitTotalWeight: number;
+  header: string;
+  utxoSetChange: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize: number | null;
+  totalInputAmt: number | null;
 }
 
+/**
+ * Note: Everything that is added in here will be automatically returned through
+ * /api/v1/block and /api/v1/blocks APIs
+ */
 export interface BlockExtended extends IEsploraApi.Block {
   extras: BlockExtension;
 }
@@ -219,12 +294,12 @@ interface RequiredParams {
 }
 
 export interface ILoadingIndicators { [name: string]: number; }
-export interface IConversionRates { [currency: string]: number; }
 
 export interface IBackendInfo {
   hostname: string;
   gitCommit: string;
   version: string;
+  lightning: boolean;
 }
 
 export interface IDifficultyAdjustment {
@@ -251,3 +326,41 @@ export interface RewardStats {
   totalFee: number;
   totalTx: number;
 }
+
+export interface ITopNodesPerChannels {
+  publicKey: string,
+  alias: string,
+  channels?: number,
+  capacity: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface ITopNodesPerCapacity {
+  publicKey: string,
+  alias: string,
+  capacity: number,
+  channels?: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface INodesRanking {
+  topByCapacity: ITopNodesPerCapacity[];
+  topByChannels: ITopNodesPerChannels[];
+}
+
+export interface IOldestNodes {
+  publicKey: string,
+  alias: string,
+  firstSeen: number,
+  channels?: number,
+  capacity: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,14 +1,14 @@
-import transactionUtils from '../api/transaction-utils';
+import blocks from '../api/blocks';
 import DB from '../database';
 import logger from '../logger';
-import { BlockAudit } from '../mempool.interfaces';
+import { BlockAudit, AuditScore } from '../mempool.interfaces';
 
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, match_rate)
-        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
-          JSON.stringify(audit.addedTxs), audit.matchRate]);
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, match_rate)
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), audit.matchRate]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@@ -52,24 +52,58 @@ class BlocksAuditRepositories {
       const [rows]: any[] = await DB.query(
         `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
         blocks.weight, blocks.tx_count,
-        transactions, template, missing_txs as missingTxs, added_txs as addedTxs, match_rate as matchRate
+        transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, match_rate as matchRate
         FROM blocks_audits
         JOIN blocks ON blocks.hash = blocks_audits.hash
         JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
         WHERE blocks_audits.hash = "${hash}"
       `);
       
-      rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
-      rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
-      rows[0].transactions = JSON.parse(rows[0].transactions);
-      rows[0].template = JSON.parse(rows[0].template);
-            
+      if (rows.length) {
+        rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
+        rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
+        rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
+        rows[0].transactions = JSON.parse(rows[0].transactions);
+        rows[0].template = JSON.parse(rows[0].template);
+
+        if (rows[0].transactions.length) {
+          return rows[0];
+        }
+      }
+      return null;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.hash = "${hash}"
+      `);
       return rows[0];
     } catch (e: any) {
       logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
+
+  public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise<AuditScore[]> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.height BETWEEN ? AND ?
+      `, [minHeight, maxHeight]);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksAuditRepositories();

backend/src/repositories/BlocksRepository.ts

@@ -1,34 +1,98 @@
-import { BlockExtended, BlockPrice } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
-import { prepareBlock } from '../utils/blocks-utils';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
 import BlocksSummariesRepository from './BlocksSummariesRepository';
 import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
+import bitcoinClient from '../api/bitcoin/bitcoin-client';
+import config from '../config';
+import chainTips from '../api/chain-tips';
+import blocks from '../api/blocks';
+import BlocksAuditsRepository from './BlocksAuditsRepository';
+
+const BLOCK_DB_FIELDS = `
+  blocks.hash AS id,
+  blocks.height,
+  blocks.version,
+  UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
+  blocks.bits,
+  blocks.nonce,
+  blocks.difficulty,
+  blocks.merkle_root,
+  blocks.tx_count,
+  blocks.size,
+  blocks.weight,
+  blocks.previous_block_hash AS previousblockhash,
+  UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
+  blocks.fees AS totalFees,
+  blocks.median_fee AS medianFee,
+  blocks.fee_span AS feeRange,
+  blocks.reward,
+  pools.unique_id AS poolId,
+  pools.name AS poolName,
+  pools.slug AS poolSlug,
+  blocks.avg_fee AS avgFee,
+  blocks.avg_fee_rate AS avgFeeRate,
+  blocks.coinbase_raw AS coinbaseRaw,
+  blocks.coinbase_address AS coinbaseAddress,
+  blocks.coinbase_signature AS coinbaseSignature,
+  blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
+  blocks.avg_tx_size AS avgTxSize,
+  blocks.total_inputs AS totalInputs,
+  blocks.total_outputs AS totalOutputs,
+  blocks.total_output_amt AS totalOutputAmt,
+  blocks.median_fee_amt AS medianFeeAmt,
+  blocks.fee_percentiles AS feePercentiles,
+  blocks.segwit_total_txs AS segwitTotalTxs,
+  blocks.segwit_total_size AS segwitTotalSize,
+  blocks.segwit_total_weight AS segwitTotalWeight,
+  blocks.header,
+  blocks.utxoset_change AS utxoSetChange,
+  blocks.utxoset_size AS utxoSetSize,
+  blocks.total_input_amt AS totalInputAmts
+`;
 
 class BlocksRepository {
   /**
    * Save indexed block data in the database
    */
   public async $saveBlockInDatabase(block: BlockExtended) {
+    const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
+    const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
+
     try {
       const query = `INSERT INTO blocks(
-        height,           hash,                blockTimestamp, size,
-        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id,          fees,                fee_span,       median_fee,
-        reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        height,             hash,                blockTimestamp,    size,
+        weight,             tx_count,            coinbase_raw,      difficulty,
+        pool_id,            fees,                fee_span,          median_fee,
+        reward,             version,             bits,              nonce,
+        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        median_timestamp,   header,              coinbase_address,
+        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        median_fee_amt,     coinbase_signature_ascii
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        FROM_UNIXTIME(?), ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?
       )`;
 
+      const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
+      if (!poolDbId) {
+        throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
+      }
+
       const params: any[] = [
         block.height,
         block.id,
@@ -38,7 +102,7 @@ class BlocksRepository {
         block.tx_count,
         block.extras.coinbaseRaw,
         block.difficulty,
-        block.extras.pool?.id, // Should always be set to something
+        poolDbId.id,
         block.extras.totalFees,
         JSON.stringify(block.extras.feeRange),
         block.extras.medianFee,
@@ -50,19 +114,63 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
+        block.mediantime,
+        block.extras.header,
+        block.extras.coinbaseAddress,
+        truncatedCoinbaseSignature,
+        block.extras.utxoSetSize,
+        block.extras.utxoSetChange,
+        block.extras.avgTxSize,
+        block.extras.totalInputs,
+        block.extras.totalOutputs,
+        block.extras.totalInputAmt,
+        block.extras.totalOutputAmt,
+        block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
+        block.extras.segwitTotalTxs,
+        block.extras.segwitTotalSize,
+        block.extras.segwitTotalWeight,
+        block.extras.medianFeeAmt,
+        truncatedCoinbaseSignatureAscii,
       ];
 
       await DB.query(query, params);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
+        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
       } else {
-        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
+        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
         throw e;
       }
     }
   }
 
+  /**
+   * Save newly indexed data from core coinstatsindex
+   * 
+   * @param utxoSetSize 
+   * @param totalInputAmt 
+   */
+  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
+    totalInputAmt: number
+  ) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET utxoset_size = ?, total_input_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        utxoSetSize,
+        totalInputAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -222,6 +330,55 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get average block health for all blocks for a single pool
+   */
+  public async $getAvgBlockHealthPerPoolId(poolId: number): Promise<number> {
+    const params: any[] = [];
+    const query = `
+      SELECT AVG(blocks_audits.match_rate) AS avg_match_rate
+      FROM blocks
+      JOIN blocks_audits ON blocks.height = blocks_audits.height
+      WHERE blocks.pool_id = ?
+    `;
+    params.push(poolId);
+
+    try {
+      const [rows] = await DB.query(query, params);
+      if (!rows[0] || !rows[0].avg_match_rate) {
+        return 0;
+      }
+      return Math.round(rows[0].avg_match_rate * 100) / 100;
+    } catch (e) {
+      logger.err(`Cannot get average block health for pool id ${poolId}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get average block health for all blocks for a single pool
+   */
+  public async $getTotalRewardForPoolId(poolId: number): Promise<number> {
+    const params: any[] = [];
+    const query = `
+      SELECT sum(reward) as total_reward
+      FROM blocks
+      WHERE blocks.pool_id = ?
+    `;
+    params.push(poolId);
+
+    try {
+      const [rows] = await DB.query(query, params);
+      if (!rows[0] || !rows[0].total_reward) {
+        return 0;
+      }
+      return rows[0].total_reward;
+    } catch (e) {
+      logger.err(`Cannot get total reward for pool id ${poolId}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get the oldest indexed block
    */
@@ -248,34 +405,17 @@ class BlocksRepository {
   /**
    * Get blocks mined by a specific mining pool
    */
-  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
+  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
     const pool = await PoolsRepository.$getPool(slug);
     if (!pool) {
       throw new Error('This mining pool does not exist ' + escape(slug));
     }
 
     const params: any[] = [];
-    let query = ` SELECT
-      blocks.height,
-      hash as id,
-      UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-      size,
-      weight,
-      tx_count,
-      coinbase_raw,
-      difficulty,
-      fees,
-      fee_span,
-      median_fee,
-      reward,
-      version,
-      bits,
-      nonce,
-      merkle_root,
-      previous_block_hash as previousblockhash,
-      avg_fee,
-      avg_fee_rate
+    let query = `
+      SELECT ${BLOCK_DB_FIELDS}
       FROM blocks
+      JOIN pools ON blocks.pool_id = pools.id
       WHERE pool_id = ?`;
     params.push(pool.id);
 
@@ -288,11 +428,11 @@ class BlocksRepository {
       LIMIT 10`;
 
     try {
-      const [rows] = await DB.query(query, params);
+      const [rows]: any[] = await DB.query(query, params);
 
       const blocks: BlockExtended[] = [];
-      for (const block of <object[]>rows) {
-        blocks.push(prepareBlock(block));
+      for (const block of rows) {
+        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
       }
 
       return blocks;
@@ -305,46 +445,21 @@ class BlocksRepository {
   /**
    * Get one block by height
    */
-  public async $getBlockByHeight(height: number): Promise<object | null> {
+  public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
     try {
-      const [rows]: any[] = await DB.query(`SELECT
-        blocks.height,
-        hash,
-        hash as id,
-        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
-        pools.id as pool_id,
-        pools.name as pool_name,
-        pools.link as pool_link,
-        pools.slug as pool_slug,
-        pools.addresses as pool_addresses,
-        pools.regexes as pool_regexes,
-        fees,
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
+      const [rows]: any[] = await DB.query(`
+        SELECT ${BLOCK_DB_FIELDS}
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
-        WHERE blocks.height = ${height}
-      `);
+        WHERE blocks.height = ?`,
+        [height]
+      );
 
       if (rows.length <= 0) {
         return null;
       }
 
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
+      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -357,10 +472,7 @@ class BlocksRepository {
   public async $getBlockByHash(hash: string): Promise<object | null> {
     try {
       const query = `
-        SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
-        pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
-        pools.addresses as pool_addresses, pools.regexes as pool_regexes,
-        previous_block_hash as previousblockhash
+        SELECT ${BLOCK_DB_FIELDS}
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
         WHERE hash = ?;
@@ -370,9 +482,8 @@ class BlocksRepository {
       if (rows.length <= 0) {
         return null;
       }
-
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
+ 
+      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
     } catch (e) {
       logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -392,6 +503,36 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get the first block at or directly after a given timestamp
+   * @param timestamp number unix time in seconds
+   * @returns The height and timestamp of a block (timestamp might vary from given timestamp)
+   */
+  public async $getBlockHeightFromTimestamp(
+    timestamp: number,
+  ): Promise<{ height: number; hash: string; timestamp: number }> {
+    try {
+      // Get first block at or after the given timestamp
+      const query = `SELECT height, hash, blockTimestamp as timestamp FROM blocks
+        WHERE blockTimestamp <= FROM_UNIXTIME(?)
+        ORDER BY blockTimestamp DESC
+        LIMIT 1`;
+      const params = [timestamp];
+      const [rows]: any[][] = await DB.query(query, params);
+      if (rows.length === 0) {
+        throw new Error(`No block was found before timestamp ${timestamp}`);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err(
+        'Cannot get block height from timestamp from the db. Reason: ' +
+          (e instanceof Error ? e.message : e),
+      );
+      throw e;
+    }
+  }
+
   /**
    * Return blocks height
    */
@@ -433,8 +574,15 @@ class BlocksRepository {
   public async $validateChain(): Promise<boolean> {
     try {
       const start = new Date().getTime();
-      const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
+      const [blocks]: any[] = await DB.query(`
+        SELECT
+          height,
+          hash,
+          previous_block_hash,
+          UNIX_TIMESTAMP(blockTimestamp) AS timestamp
+        FROM blocks
+        ORDER BY height
+      `);
 
       let partialMsg = false;
       let idx = 1;
@@ -632,6 +780,39 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get a list of blocks that have not had CPFP data indexed
+   */
+   public async $getCPFPUnindexedBlocks(): Promise<any[]> {
+    try {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      const currentBlockHeight = blockchainInfo.blocks;
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight);
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+      const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
+      const [rows]: any[] = await DB.query(`
+        SELECT height
+        FROM compact_cpfp_clusters
+        WHERE height <= ? AND height >= ?
+        GROUP BY height
+        ORDER BY height DESC;
+      `, [currentBlockHeight, minHeight]);
+
+      const indexedHeights = {};
+      rows.forEach((row) => { indexedHeights[row.height] = true; });
+      const allHeights: number[] = Array.from(Array(currentBlockHeight - minHeight + 1).keys(), n => n + minHeight).reverse();
+      const unindexedHeights = allHeights.filter(x => !indexedHeights[x]);
+
+      return unindexedHeights;
+    } catch (e) {
+      logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Return the oldest block  from a consecutive chain of block from the most recent one
    */
@@ -676,7 +857,7 @@ class BlocksRepository {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`
+        query += ` (${price.height}, ${price.priceId}),`;
       }
       query = query.slice(0, -1);
       await DB.query(query);
@@ -689,6 +870,132 @@ class BlocksRepository {
       }
     }
   }
+
+  /**
+   * Get all indexed blocsk with missing coinstatsindex data
+   */
+  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash
+        FROM blocks
+        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
+          (utxoset_size IS NULL OR total_input_amt IS NULL)
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save indexed median fee to avoid recomputing it later
+   * 
+   * @param id 
+   * @param feePercentiles 
+   */
+  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?`,
+        [JSON.stringify(feePercentiles), feePercentiles[3], id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Convert a mysql row block into a BlockExtended. Note that you
+   * must provide the correct field into dbBlk object param
+   * 
+   * @param dbBlk 
+   */
+  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
+    const blk: Partial<BlockExtended> = {};
+    const extras: Partial<BlockExtension> = {};
+
+    // IEsploraApi.Block
+    blk.id = dbBlk.id;
+    blk.height = dbBlk.height;
+    blk.version = dbBlk.version;
+    blk.timestamp = dbBlk.timestamp;
+    blk.bits = dbBlk.bits;
+    blk.nonce = dbBlk.nonce;
+    blk.difficulty = dbBlk.difficulty;
+    blk.merkle_root = dbBlk.merkle_root;
+    blk.tx_count = dbBlk.tx_count;
+    blk.size = dbBlk.size;
+    blk.weight = dbBlk.weight;
+    blk.previousblockhash = dbBlk.previousblockhash;
+    blk.mediantime = dbBlk.mediantime;
+    
+    // BlockExtension
+    extras.totalFees = dbBlk.totalFees;
+    extras.medianFee = dbBlk.medianFee;
+    extras.feeRange = JSON.parse(dbBlk.feeRange);
+    extras.reward = dbBlk.reward;
+    extras.pool = {
+      id: dbBlk.poolId,
+      name: dbBlk.poolName,
+      slug: dbBlk.poolSlug,
+    };
+    extras.avgFee = dbBlk.avgFee;
+    extras.avgFeeRate = dbBlk.avgFeeRate;
+    extras.coinbaseRaw = dbBlk.coinbaseRaw;
+    extras.coinbaseAddress = dbBlk.coinbaseAddress;
+    extras.coinbaseSignature = dbBlk.coinbaseSignature;
+    extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
+    extras.avgTxSize = dbBlk.avgTxSize;
+    extras.totalInputs = dbBlk.totalInputs;
+    extras.totalOutputs = dbBlk.totalOutputs;
+    extras.totalOutputAmt = dbBlk.totalOutputAmt;
+    extras.medianFeeAmt = dbBlk.medianFeeAmt;
+    extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
+    extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
+    extras.segwitTotalSize = dbBlk.segwitTotalSize;
+    extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
+    extras.header = dbBlk.header,
+    extras.utxoSetChange = dbBlk.utxoSetChange;
+    extras.utxoSetSize = dbBlk.utxoSetSize;
+    extras.totalInputAmt = dbBlk.totalInputAmt;
+    extras.virtualSize = dbBlk.weight / 4.0;
+
+    // Re-org can happen after indexing so we need to always get the
+    // latest state from core
+    extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
+
+    // Match rate is not part of the blocks table, but it is part of APIs so we must include it
+    extras.matchRate = null;
+    if (config.MEMPOOL.AUDIT) {
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id);
+      if (auditScore != null) {
+        extras.matchRate = auditScore.matchRate;
+      }
+    }
+
+    // If we're missing block summary related field, check if we can populate them on the fly now
+    if (Common.blocksSummariesIndexingEnabled() &&
+      (extras.medianFeeAmt === null || extras.feePercentiles === null))
+    {
+      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
+      if (extras.feePercentiles === null) {
+        const block = await bitcoinClient.getBlock(dbBlk.id, 2);
+        const summary = blocks.summarizeBlock(block);
+        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+        extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
+      }
+      if (extras.feePercentiles !== null) {
+        extras.medianFeeAmt = extras.feePercentiles[3];
+      }
+    }
+
+    blk.extras = <BlockExtension>extras;
+    return <BlockExtended>blk;
+  }
 }
 
 export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -17,19 +17,16 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) {
-    const blockId = params.mined?.id ?? params.template?.id;
+  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
+    const blockId = params.mined?.id;
     try {
-      const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`);
-      if (dbSummary.length === 0) { // First insertion
-        await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [
-          params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? [])
-        ]);
-      } else if (params.mined !== undefined) { // Update mined block summary
-        await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]);
-      } else if (params.template !== undefined) { // Update template block summary
-        await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]);
-      }
+      const transactions = JSON.stringify(params.mined?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          transactions = ?
+      `, [params.height, blockId, transactions, '[]', transactions]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
@@ -40,6 +37,26 @@ class BlocksSummariesRepository {
     }
   }
 
+  public async $saveTemplate(params: { height: number, template: BlockSummary}) {
+    const blockId = params.template?.id;
+    try {
+      const transactions = JSON.stringify(params.template?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          template = ?
+      `, [params.height, blockId, '[]', transactions, transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
   public async $getIndexedSummariesId(): Promise<string[]> {
     try {
       const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);
@@ -63,6 +80,48 @@ class BlocksSummariesRepository {
       logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0] ?? 0, // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
+        fees[fees.length - 1] ?? 0, // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();

backend/src/repositories/CpfpRepository.ts

@@ -0,0 +1,232 @@
+import cluster, { Cluster } from 'cluster';
+import { RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+import { Ancestor } from '../mempool.interfaces';
+import transactionRepository from '../repositories/TransactionRepository';
+
+class CpfpRepository {
+  public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<boolean> {
+    if (!txs[0]) {
+      return false;
+    }
+    // skip clusters of transactions with the same fees
+    const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
+    const equalFee = txs.reduce((acc, tx) => {
+      return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+    }, true);
+    if (equalFee) {
+      return false;
+    }
+
+    try {
+      const packedTxs = Buffer.from(this.pack(txs));
+      await DB.query(
+        `
+          INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+          VALUE (UNHEX(?), ?, ?, ?)
+          ON DUPLICATE KEY UPDATE
+            height = ?,
+            txs = ?,
+            fee_rate = ?
+        `,
+        [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize]
+      );
+      const maxChunk = 10;
+      let chunkIndex = 0;
+      while (chunkIndex < txs.length) {
+        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => {
+          return { txid: tx.txid, cluster: clusterRoot };
+        });
+        await transactionRepository.$batchSetCluster(chunk);
+        chunkIndex += maxChunk;
+      }
+      return true;
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
+    try {
+      const clusterValues: any[] = [];
+      const txs: any[] = [];
+
+      for (const cluster of clusters) {
+        if (cluster.txs?.length > 1) {
+          const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
+          const equalFee = cluster.txs.reduce((acc, tx) => {
+            return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+          }, true);
+          if (!equalFee) {
+            clusterValues.push([
+              cluster.root,
+              cluster.height,
+              Buffer.from(this.pack(cluster.txs)),
+              cluster.effectiveFeePerVsize
+            ]);
+            for (const tx of cluster.txs) {
+              txs.push({ txid: tx.txid, cluster: cluster.root });
+            }
+          }
+        }
+      }
+
+      if (!clusterValues.length) {
+        return false;
+      }
+
+      const maxChunk = 100;
+      let chunkIndex = 0;
+      // insert transactions in batches of up to 100 rows
+      while (chunkIndex < txs.length) {
+        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk);
+        await transactionRepository.$batchSetCluster(chunk);
+        chunkIndex += maxChunk;
+      }
+
+      chunkIndex = 0;
+      // insert clusters in batches of up to 100 rows
+      while (chunkIndex < clusterValues.length) {
+        const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk);
+        let query = `
+            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+            VALUES
+        `;
+        query += chunk.map(chunk => {
+          return (' (UNHEX(?), ?, ?, ?)');
+        }) + ';';
+        const values = chunk.flat();
+        await DB.query(
+          query,
+          values
+        );
+        chunkIndex += maxChunk;
+      }
+      return true;
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getCluster(clusterRoot: string): Promise<Cluster | void> {
+    const [clusterRows]: any = await DB.query(
+      `
+        SELECT *
+        FROM compact_cpfp_clusters
+        WHERE root = UNHEX(?)
+      `,
+      [clusterRoot]
+    );
+    const cluster = clusterRows[0];
+    if (cluster?.txs) {
+      cluster.txs = this.unpack(cluster.txs);
+      return cluster;
+    }
+    return;
+  }
+
+  public async $deleteClustersFrom(height: number): Promise<void> {
+    logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
+    try {
+      const [rows] = await DB.query(
+        `
+          SELECT txs, height, root from compact_cpfp_clusters
+          WHERE height >= ?
+        `,
+        [height]
+      ) as RowDataPacket[][];
+      if (rows?.length) {
+        for (const clusterToDelete of rows) {
+          const txs = this.unpack(clusterToDelete?.txs);
+          for (const tx of txs) {
+            await transactionRepository.$removeTransaction(tx.txid);
+          }
+        }
+      }
+      await DB.query(
+        `
+          DELETE from compact_cpfp_clusters
+          WHERE height >= ?
+        `,
+        [height]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  // insert a dummy row to mark that we've indexed as far as this block
+  public async $insertProgressMarker(height: number): Promise<void> {
+    try {
+      const [rows]: any = await DB.query(
+        `
+          SELECT root
+          FROM compact_cpfp_clusters
+          WHERE height = ?
+        `,
+        [height]
+      );
+      if (!rows?.length) {
+        const rootBuffer = Buffer.alloc(32);
+        rootBuffer.writeInt32LE(height);
+        await DB.query(
+          `
+            INSERT INTO compact_cpfp_clusters(root, height, fee_rate)
+            VALUE (?, ?, ?)
+          `,
+          [rootBuffer, height, 0]
+        );
+      }
+    } catch (e: any) {
+      logger.err(`Cannot insert cpfp progress marker. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public pack(txs: Ancestor[]): ArrayBuffer {
+    const buf = new ArrayBuffer(44 * txs.length);
+    const view = new DataView(buf);
+    txs.forEach((tx, i) => {
+      const offset = i * 44;
+      for (let x = 0; x < 32; x++) {
+        // store txid in little-endian
+        view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16));
+      }
+      view.setUint32(offset + 32, tx.weight);
+      view.setBigUint64(offset + 36, BigInt(Math.round(tx.fee)));
+    });
+    return buf;
+  }
+
+  public unpack(buf: Buffer): Ancestor[] {
+    if (!buf) {
+      return [];
+    }
+
+    try {
+      const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+      const txs: Ancestor[] = [];
+      const view = new DataView(arrayBuffer);
+      for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
+        const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
+        const weight = view.getUint32(offset + 32);
+        const fee = Number(view.getBigUint64(offset + 36));
+        txs.push({
+          txid,
+          weight,
+          fee
+        });
+      }
+      return txs;
+    } catch (e) {
+      logger.warn(`Failed to unpack CPFP cluster. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+}
+
+export default new CpfpRepository();

backend/src/repositories/DifficultyAdjustmentsRepository.ts

@@ -1,5 +1,4 @@
 import { Common } from '../api/common';
-import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
@@ -21,9 +20,9 @@ class DifficultyAdjustmentsRepository {
       await DB.query(query, params);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
+        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`, logger.tags.mining);
       } else {
-        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
+        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
         throw e;
       }
     }
@@ -55,7 +54,7 @@ class DifficultyAdjustmentsRepository {
       const [rows] = await DB.query(query);
       return rows as IndexedDifficultyAdjustment[];
     } catch (e) {
-      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -84,7 +83,7 @@ class DifficultyAdjustmentsRepository {
       const [rows] = await DB.query(query);
       return rows as IndexedDifficultyAdjustment[];
     } catch (e) {
-      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -94,27 +93,27 @@ class DifficultyAdjustmentsRepository {
       const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
       return rows.map(block => block.height);
     } catch (e: any) {
-      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
       throw e;
     }
   }
 
   public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
     try {
-      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
+      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`, logger.tags.mining);
       await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
     } catch (e: any) {
-      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
       throw e;
     }
   }
 
   public async $deleteLastAdjustment(): Promise<void> {
     try {
-      logger.info(`Delete last difficulty adjustment from the database`);
+      logger.info(`Delete last difficulty adjustment from the database`, logger.tags.mining);
       await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
     } catch (e: any) {
-      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
       throw e;
     }
   }

backend/src/repositories/HashratesRepository.ts

@@ -1,5 +1,6 @@
 import { escape } from 'mysql2';
 import { Common } from '../api/common';
+import mining from '../api/mining/mining';
 import DB from '../database';
 import logger from '../logger';
 import PoolsRepository from './PoolsRepository';
@@ -24,7 +25,7 @@ class HashratesRepository {
     try {
       await DB.query(query);
     } catch (e: any) {
-      logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -50,7 +51,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -77,7 +78,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -92,7 +93,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows.map(row => row.timestamp);
     } catch (e) {
-      logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -127,7 +128,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -157,7 +158,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query, [pool.id]);
       boundaries = rows[0];
     } catch (e) {
-      logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
     }
 
     // Get hashrates entries between boundaries
@@ -172,21 +173,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Set latest run timestamp
-   */
-  public async $setLatestRun(key: string, val: number) {
-    const query = `UPDATE state SET number = ? WHERE name = ?`;
-
-    try {
-      await DB.query(query, [val, key]);
-    } catch (e) {
-      logger.err(`Cannot set last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -205,7 +192,7 @@ class HashratesRepository {
       }
       return rows[0]['number'];
     } catch (e) {
-      logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
+      logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
       throw e;
     }
   }
@@ -214,7 +201,7 @@ class HashratesRepository {
    * Delete most recent data points for re-indexing
    */
   public async $deleteLastEntries() {
-    logger.info(`Delete latest hashrates data points from the database`);
+    logger.info(`Delete latest hashrates data points from the database`, logger.tags.mining);
 
     try {
       const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
@@ -222,10 +209,10 @@ class HashratesRepository {
         await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
       }
       // Re-run the hashrate indexing to fill up missing data
-      await this.$setLatestRun('last_hashrates_indexing', 0);
-      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
+      mining.lastHashrateIndexingDate = null;
+      mining.lastWeeklyHashrateIndexingDate = null;
     } catch (e) {
-      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
     }
   }
   
@@ -238,10 +225,10 @@ class HashratesRepository {
     try {
       await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);
       // Re-run the hashrate indexing to fill up missing data
-      await this.$setLatestRun('last_hashrates_indexing', 0);
-      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
+      mining.lastHashrateIndexingDate = null;
+      mining.lastWeeklyHashrateIndexingDate = null;
     } catch (e) {
-      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
     }
   }
 }

backend/src/repositories/NodeRecordsRepository.ts

@@ -0,0 +1,67 @@
+import { ResultSetHeader, RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeRecord {
+  publicKey: string; // node public key
+  type: number; // TLV extension record type
+  payload: string; // base64 record payload
+}
+
+class NodesRecordsRepository {
+  public async $saveRecord(record: NodeRecord): Promise<void> {
+    try {
+      const payloadBytes = Buffer.from(record.payload, 'base64');
+      await DB.query(`
+        INSERT INTO nodes_records(public_key, type, payload)
+        VALUE (?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          payload = ?
+      `, [record.publicKey, record.type, payloadBytes, payloadBytes]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node record (${[record.publicKey, record.type, record.payload]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes records
+      }
+    }
+   }
+
+  public async $getRecordTypes(publicKey: string): Promise<any> {
+    try {
+      const query = `
+        SELECT type FROM nodes_records
+        WHERE public_key = ?
+      `;
+      const [rows] = await DB.query<RowDataPacket[][]>(query, [publicKey]);
+      return rows.map(row => row['type']);
+    } catch (e) {
+      logger.err(`Cannot retrieve custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise<number> {
+    try {
+      let query;
+      if (recordTypes.length) {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+          AND type NOT IN (${recordTypes.map(type => `${type}`).join(',')})
+        `;
+      } else {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+        `;
+      }
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+  }
+}
+
+export default new NodesRecordsRepository();

backend/src/repositories/NodesSocketsRepository.ts

@@ -0,0 +1,45 @@
+import { ResultSetHeader } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeSocket {
+  publicKey: string;
+  network: string | null;
+  addr: string;
+}
+
+class NodesSocketsRepository {
+  public async $saveSocket(socket: NodeSocket): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO nodes_sockets(public_key, socket, type)
+        VALUE (?, ?, ?)
+      `, [socket.publicKey, socket.addr, socket.network]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes sockets
+      }
+    }
+   }
+
+   public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
+    if (addresses.length === 0) {
+      return 0;
+    }
+    try {
+      const query = `
+        DELETE FROM nodes_sockets
+        WHERE public_key = ?
+        AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
+      `;
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+   }
+}
+
+export default new NodesSocketsRepository();

backend/src/repositories/PoolsRepository.ts

@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
@@ -9,7 +10,7 @@ class PoolsRepository {
    * Get all pools tagging info
    */
   public async $getPools(): Promise<PoolTag[]> {
-    const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
+    const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
     return <PoolTag[]>rows;
   }
 
@@ -17,7 +18,11 @@ class PoolsRepository {
    * Get unknown pool tagging info
    */
   public async $getUnknownPool(): Promise<PoolTag> {
-    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
+    if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
+      await poolsParser.$insertUnknownPool();
+      [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
+    }
     return <PoolTag>rows[0];
   }
 
@@ -27,16 +32,25 @@ class PoolsRepository {
   public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
+    let query = `
+      SELECT
+        COUNT(blocks.height) As blockCount,
+          pool_id AS poolId,
+          pools.name AS name,
+          pools.link AS link,
+          slug,
+          AVG(blocks_audits.match_rate) AS avgMatchRate
       FROM blocks
-      JOIN pools on pools.id = pool_id`;
+      JOIN pools on pools.id = pool_id
+      LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
+    `;
 
     if (interval) {
       query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
     }
 
     query += ` GROUP BY pool_id
-      ORDER BY COUNT(height) DESC`;
+      ORDER BY COUNT(blocks.height) DESC`;
 
     try {
       const [rows] = await DB.query(query);
@@ -50,7 +64,7 @@ class PoolsRepository {
   /**
    * Get basic pool info and block count between two timestamp
    */
-   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+  public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
     const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
       FROM pools
       LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
@@ -66,9 +80,9 @@ class PoolsRepository {
   }
 
   /**
-   * Get mining pool statistics for one pool
+   * Get a mining pool info
    */
-   public async $getPool(slug: string): Promise<PoolTag | null> {
+  public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
     const query = `
       SELECT *
       FROM pools
@@ -81,10 +95,12 @@ class PoolsRepository {
         return null;
       }
 
-      rows[0].regexes = JSON.parse(rows[0].regexes);
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
       if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools.json only contains mainnet addresses
-      } else {
+        rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
+      } else if (parse) {
         rows[0].addresses = JSON.parse(rows[0].addresses);
       }
 
@@ -94,6 +110,116 @@ class PoolsRepository {
       throw e;
     }
   }
+
+  /**
+   * Get a mining pool info by its unique id
+   */
+  public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.unique_id = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [id]);
+
+      if (rows.length < 1) {
+        return null;
+      }
+
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+        rows[0].addresses = []; // pools.json only contains mainnet addresses
+      } else if (parse) {
+        rows[0].addresses = JSON.parse(rows[0].addresses);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Insert a new mining pool in the database
+   * 
+   * @param pool 
+   */
+  public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO pools
+        SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`,
+        [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Rename an existing mining pool
+   * 
+   * @param dbId
+   * @param newSlug
+   * @param newName 
+   */
+  public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET slug = ?, name = ?
+        WHERE id = ?`,
+        [newSlug, newName, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update an exisiting mining pool link
+   * 
+   * @param dbId 
+   * @param newLink 
+   */
+  public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET link = ?
+        WHERE id = ?`,
+        [newLink, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+  }
+
+  /**
+   * Update an existing mining pool addresses or coinbase tags
+   * 
+   * @param dbId 
+   * @param addresses 
+   * @param regexes 
+   */
+  public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET addresses = ?, regexes = ?
+        WHERE id = ?`,
+        [JSON.stringify(addresses), JSON.stringify(regexes), dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
 }
 
 export default new PoolsRepository();

backend/src/repositories/PricesRepository.ts

@@ -1,45 +1,228 @@
 import DB from '../database';
 import logger from '../logger';
-import { Prices } from '../tasks/price-updater';
+import priceUpdater from '../tasks/price-updater';
+
+export interface ApiPrice {
+  time?: number,
+  USD: number,
+  EUR: number,
+  GBP: number,
+  CAD: number,
+  CHF: number,
+  AUD: number,
+  JPY: number,
+}
+const ApiPriceFields = `
+  UNIX_TIMESTAMP(time) as time,
+  USD,
+  EUR,
+  GBP,
+  CAD,
+  CHF,
+  AUD,
+  JPY
+`;
+
+export interface ExchangeRates {
+  USDEUR: number,
+  USDGBP: number,
+  USDCAD: number,
+  USDCHF: number,
+  USDAUD: number,
+  USDJPY: number,
+}
+
+export interface Conversion {
+  prices: ApiPrice[],
+  exchangeRates: ExchangeRates;
+}
+
+export const MAX_PRICES = {
+  USD: 100000000,
+  EUR: 100000000,
+  GBP: 100000000,
+  CAD: 100000000,
+  CHF: 100000000,
+  AUD: 100000000,
+  JPY: 10000000000,
+};
 
 class PricesRepository {
-  public async $savePrices(time: number, prices: Prices): Promise<void> {
+  public async $savePrices(time: number, prices: ApiPrice): Promise<void> {
     if (prices.USD === -1) {
-      // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
-      // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
+      // Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues
+      // As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine
       return;
     }
 
+    // Sanity check
+    for (const currency of Object.keys(prices)) {
+      if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry"
+        logger.info(`Ignore BTC${currency} price of ${prices[currency]}`);
+        prices[currency] = 0;
+      }
+    }
+    
     try {
       await DB.query(`
         INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
         VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
         [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
       );
-    } catch (e: any) {
+    } catch (e) {
       logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
   public async $getOldestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
+    const [oldestRow] = await DB.query(`
+      SELECT UNIX_TIMESTAMP(time) AS time
+      FROM prices
+      ORDER BY time
+      LIMIT 1
+    `);
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
+  public async $getLatestPriceId(): Promise<number | null> {
+    const [oldestRow] = await DB.query(`
+      SELECT id
+      FROM prices
+      ORDER BY time DESC
+      LIMIT 1`
+    );
+    return oldestRow[0] ? oldestRow[0].id : null;
+  }
+
   public async $getLatestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    const [oldestRow] = await DB.query(`
+      SELECT UNIX_TIMESTAMP(time) AS time
+      FROM prices
+      ORDER BY time DESC
+      LIMIT 1`
+    );
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
   public async $getPricesTimes(): Promise<number[]> {
-    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
+    const [times] = await DB.query(`
+      SELECT UNIX_TIMESTAMP(time) AS time
+      FROM prices
+      WHERE USD != -1
+      ORDER BY time
+    `);
+    if (!Array.isArray(times)) {
+      return [];
+    }
     return times.map(time => time.time);
   }
 
-  public async $getPricesTimesAndId(): Promise<number[]> {
-    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
-    return times;
+  public async $getPricesTimesAndId(): Promise<{time: number, id: number, USD: number}[]> {
+    const [times] = await DB.query(`
+      SELECT
+        UNIX_TIMESTAMP(time) AS time,
+        id,
+        USD
+      FROM prices
+      ORDER BY time
+    `);
+    return times as {time: number, id: number, USD: number}[];
+  }
+
+  public async $getLatestConversionRates(): Promise<ApiPrice> {
+    const [rates] = await DB.query(`
+      SELECT ${ApiPriceFields}
+      FROM prices
+      ORDER BY time DESC
+      LIMIT 1`
+    );
+
+    if (!Array.isArray(rates) || rates.length === 0) {
+      return priceUpdater.getEmptyPricesObj();
+    }
+    return rates[0] as ApiPrice;
+  }
+
+  public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
+    try {
+      const [rates] = await DB.query(`
+        SELECT ${ApiPriceFields}
+        FROM prices
+        WHERE UNIX_TIMESTAMP(time) < ?
+        ORDER BY time DESC
+        LIMIT 1`,
+        [timestamp]
+      );
+      if (!Array.isArray(rates)) {
+        throw Error(`Cannot get single historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      let latestPrice = rates[0] as ApiPrice;
+      if (latestPrice.USD === -1) {
+        latestPrice = priceUpdater.getEmptyPricesObj();
+      }
+
+      const computeFx = (usd: number, other: number): number =>
+        Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
+      
+      const exchangeRates: ExchangeRates = {
+        USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
+        USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
+        USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
+        USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
+        USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
+        USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
+      };
+
+      return {
+        prices: rates as ApiPrice[],
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
+
+  public async $getHistoricalPrices(): Promise<Conversion | null> {
+    try {
+      const [rates] = await DB.query(`
+        SELECT ${ApiPriceFields}
+        FROM prices
+        ORDER BY time DESC
+      `);
+      if (!Array.isArray(rates)) {
+        throw Error(`Cannot get average historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      let latestPrice = rates[0] as ApiPrice;
+      if (latestPrice.USD === -1) {
+        latestPrice = priceUpdater.getEmptyPricesObj();
+      }
+
+      const computeFx = (usd: number, other: number): number =>
+        Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
+      
+      const exchangeRates: ExchangeRates = {
+        USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
+        USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
+        USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
+        USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
+        USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
+        USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
+      };
+
+      return {
+        prices: rates as ApiPrice[],
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
   }
 }
 

backend/src/repositories/TransactionRepository.ts

@@ -0,0 +1,113 @@
+import DB from '../database';
+import logger from '../logger';
+import { Ancestor, CpfpInfo } from '../mempool.interfaces';
+import cpfpRepository from './CpfpRepository';
+
+class TransactionRepository {
+  public async $setCluster(txid: string, clusterRoot: string): Promise<void> {
+    try {
+      await DB.query(
+        `
+          INSERT INTO compact_transactions
+          (
+            txid,
+            cluster
+          )
+          VALUE (UNHEX(?), UNHEX(?))
+          ON DUPLICATE KEY UPDATE
+            cluster = UNHEX(?)
+        ;`,
+        [txid, clusterRoot, clusterRoot]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $batchSetCluster(txs): Promise<void> {
+    try {
+      let query = `
+          INSERT IGNORE INTO compact_transactions
+          (
+            txid,
+            cluster
+          )
+          VALUES
+      `;
+      query += txs.map(tx => {
+        return (' (UNHEX(?), UNHEX(?))');
+      }) + ';';
+      const values = txs.map(tx => [tx.txid, tx.cluster]).flat();
+      await DB.query(
+        query,
+        values
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
+    try {
+      const [txRows]: any = await DB.query(
+        `
+          SELECT HEX(txid) as id, HEX(cluster) as root
+          FROM compact_transactions
+          WHERE txid = UNHEX(?)
+        `,
+        [txid]
+      );
+      if (txRows.length && txRows[0].root != null) {
+        const txid = txRows[0].id.toLowerCase();
+        const clusterId = txRows[0].root.toLowerCase();
+        const cluster = await cpfpRepository.$getCluster(clusterId);
+        if (cluster) {
+          return this.convertCpfp(txid, cluster);
+        }
+      }
+    } catch (e) {
+      logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $removeTransaction(txid: string): Promise<void> {
+    try {
+      await DB.query(
+        `
+          DELETE FROM compact_transactions
+          WHERE txid = UNHEX(?)
+        `,
+        [txid]
+      );
+    } catch (e) {
+      logger.warn('Cannot delete transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private convertCpfp(txid, cluster): CpfpInfo {
+    const descendants: Ancestor[] = [];
+    const ancestors: Ancestor[] = [];
+    let matched = false;
+
+    for (const tx of (cluster?.txs || [])) {
+      if (tx.txid === txid) {
+        matched = true;
+      } else if (!matched) {
+        descendants.push(tx);
+      } else {
+        ancestors.push(tx);
+      }
+    }
+    return {
+      descendants,
+      ancestors,
+    };
+  }
+}
+
+export default new TransactionRepository();
+

backend/src/rpc-api/commands.ts

@@ -88,5 +88,7 @@ module.exports = {
   verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange'
-}
+  walletPassphraseChange: 'walletpassphrasechange',
+  getTxoutSetinfo: 'gettxoutsetinfo',
+  getIndexInfo: 'getindexinfo',
+};

backend/src/tasks/lightning/forensics.service.ts

@@ -0,0 +1,457 @@
+import DB from '../../database';
+import logger from '../../logger';
+import channelsApi from '../../api/explorer/channels.api';
+import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
+import config from '../../config';
+import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
+import { Common } from '../../api/common';
+import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+
+const tempCacheSize = 10000;
+
+class ForensicsService {
+  loggerTimer = 0;
+  closedChannelsScanBlock = 0;
+  txCache: { [txid: string]: IEsploraApi.Transaction } = {};
+  tempCached: string[] = [];
+
+  constructor() {}
+
+  public async $startService(): Promise<void> {
+    logger.info('Starting lightning network forensics service');
+
+    this.loggerTimer = new Date().getTime() / 1000;
+
+    await this.$runTasks();
+  }
+
+  private async $runTasks(): Promise<void> {
+    try {
+      logger.info(`Running forensics scans`);
+
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        await this.$runClosedChannelsForensics(false);
+        await this.$runOpenedChannelsForensics();
+      }
+
+    } catch (e) {
+      logger.err('ForensicsService.$runTasks() error: ' + (e instanceof Error ? e.message : e));
+    }
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.FORENSICS_INTERVAL);
+  }
+
+  /*
+    1. Mutually closed
+    2. Forced closed
+    3. Forced closed with penalty
+
+    ┌────────────────────────────────────┐       ┌────────────────────────────┐
+    │ outputs contain revocation script? ├──yes──► force close w/ penalty = 3 │
+    └──────────────┬─────────────────────┘       └────────────────────────────┘
+                   no
+    ┌──────────────▼──────────────────────────┐
+    │ outputs contain other lightning script? ├──┐
+    └──────────────┬──────────────────────────┘  │
+                   no                           yes
+    ┌──────────────▼─────────────┐               │
+    │ sequence starts with 0x80  │      ┌────────▼────────┐
+    │           and              ├──────► force close = 2 │
+    │ locktime starts with 0x20? │      └─────────────────┘
+    └──────────────┬─────────────┘
+                   no
+         ┌─────────▼────────┐
+         │ mutual close = 1 │
+         └──────────────────┘
+  */
+
+  public async $runClosedChannelsForensics(onlyNewChannels: boolean = false): Promise<void> {
+    if (config.MEMPOOL.BACKEND !== 'esplora') {
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      logger.info(`Started running closed channel forensics...`);
+      let channels;
+      if (onlyNewChannels) {
+        channels = await channelsApi.$getClosedChannelsWithoutReason();
+      } else {
+        channels = await channelsApi.$getUnresolvedClosedChannels();
+      }
+
+      for (const channel of channels) {
+        let reason = 0;
+        let resolvedForceClose = false;
+        // Only Esplora backend can retrieve spent transaction outputs
+        const cached: string[] = [];
+        try {
+          let outspends: IEsploraApi.Outspend[] | undefined;
+          try {
+            outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
+            await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+          } catch (e) {
+            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+            continue;
+          }
+          const lightningScriptReasons: number[] = [];
+          for (const outspend of outspends) {
+            if (outspend.spent && outspend.txid) {
+              let spendingTx = await this.fetchTransaction(outspend.txid);
+              if (!spendingTx) {
+                continue;
+              }
+              cached.push(spendingTx.txid);
+              const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              lightningScriptReasons.push(lightningScript);
+            }
+          }
+          const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
+          if (filteredReasons.length) {
+            if (filteredReasons.some((r) => r === 2 || r === 4)) {
+              reason = 3;
+            } else {
+              reason = 2;
+              resolvedForceClose = true;
+            }
+          } else {
+            /*
+              We can detect a commitment transaction (force close) by reading Sequence and Locktime
+              https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
+            */
+            let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true);
+            if (!closingTx) {
+              continue;
+            }
+            cached.push(closingTx.txid);
+            const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
+            const locktimeHex: string = closingTx.locktime.toString(16);
+            if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
+              reason = 2; // Here we can't be sure if it's a penalty or not
+            } else {
+              reason = 1;
+            }
+          }
+          if (reason) {
+            logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
+            await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+            if (reason === 2 && resolvedForceClose) {
+              await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]);
+            }
+            if (reason !== 2 || resolvedForceClose) {
+              cached.forEach(txid => {
+                delete this.txCache[txid];
+              });
+            }
+          }
+        } catch (e) {
+          logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Closed channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private findLightningScript(vin: IEsploraApi.Vin): number {
+    const topElement = vin.witness?.length > 2 ? vin.witness[vin.witness.length - 2] : null;
+      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
+        if (topElement === '01') {
+          // top element is '01' to get in the revocation path
+          // 'Revoked Lightning Force Close';
+          // Penalty force closed
+          return 2;
+        } else {
+          // top element is '', this is a delayed to_local output
+          // 'Lightning Force Close';
+          return 3;
+        }
+      } else if (
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
+      ) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
+        if (topElement?.length === 66) {
+          // top element is a public key
+          // 'Revoked Lightning HTLC'; Penalty force closed
+          return 4;
+        } else if (topElement) {
+          // top element is a preimage
+          // 'Lightning HTLC';
+          return 5;
+        } else {
+          // top element is '' to get in the expiry of the script
+          // 'Expired Lightning HTLC';
+          return 6;
+        }
+      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
+        if (topElement) {
+          // top element is a signature
+          // 'Lightning Anchor';
+          return 7;
+        } else {
+          // top element is '', it has been swept after 16 blocks
+          // 'Swept Lightning Anchor';
+          return 8;
+        }
+      }
+      return 1;
+  }
+
+  // If a channel open tx spends funds from a another channel transaction,
+  // we can attribute that output to a specific counterparty
+  private async $runOpenedChannelsForensics(): Promise<void> {
+    const runTimer = Date.now();
+    let progress = 0;
+
+    try {
+      logger.info(`Started running open channel forensics...`);
+      const channels = await channelsApi.$getChannelsWithoutSourceChecked();
+
+      for (const openChannel of channels) {
+        let openTx = await this.fetchTransaction(openChannel.transaction_id, true);
+        if (!openTx) {
+          continue;
+        }
+        for (const input of openTx.vin) {
+          const closeChannel = await channelsApi.$getChannelByClosingId(input.txid);
+          if (closeChannel) {
+            // this input directly spends a channel close output
+            await this.$attributeChannelBalances(closeChannel, openChannel, input);
+          } else {
+            const prevOpenChannels = await channelsApi.$getChannelsByOpeningId(input.txid);
+            if (prevOpenChannels?.length) {
+              // this input spends a channel open change output
+              for (const prevOpenChannel of prevOpenChannels) {
+                await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true);
+              }
+            } else {
+              // check if this input spends any swept channel close outputs
+              await this.$attributeSweptChannelCloses(openChannel, input);
+            }
+          }
+        }
+        // calculate how much of the total input value is attributable to the channel open output
+        openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee);
+        // save changes to the opening channel, and mark it as checked
+        if (openTx?.vin?.length === 1) {
+          openChannel.single_funded = true;
+        }
+        if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) {
+          await channelsApi.$updateOpeningInfo(openChannel);
+        }
+        await channelsApi.$markChannelSourceChecked(openChannel.id);
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+          this.truncateTempCache();
+        }
+        if (Date.now() - runTimer > (config.LIGHTNING.FORENSICS_INTERVAL * 1000)) {
+          break;
+        }
+      }
+
+      logger.info(`Open channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    } finally {
+      this.clearTempCache();
+    }
+  }
+
+  // Check if a channel open tx input spends the result of a swept channel close output
+  private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
+    let sweepTx = await this.fetchTransaction(input.txid, true);
+    if (!sweepTx) {
+      logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
+      return;
+    }
+    const openContribution = sweepTx.vout[input.vout].value;
+    for (const sweepInput of sweepTx.vin) {
+      const lnScriptType = this.findLightningScript(sweepInput);
+      if (lnScriptType > 1) {
+        const closeChannel = await channelsApi.$getChannelByClosingId(sweepInput.txid);
+        if (closeChannel) {
+          const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null);
+          await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator);
+        }
+      }
+    }
+  }
+
+  private async $attributeChannelBalances(
+    prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
+    initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false
+  ): Promise<void> {
+    // figure out which node controls the input/output
+    let openSide;
+    let prevLocal;
+    let prevRemote;
+    let matched = false;
+    let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart
+    if (openChannel.node1_public_key === prevChannel.node1_public_key) {
+      openSide = 1;
+      prevLocal = 1;
+      prevRemote = 2;
+      matched = true;
+    } else if (openChannel.node1_public_key === prevChannel.node2_public_key) {
+      openSide = 1;
+      prevLocal = 2;
+      prevRemote = 1;
+      matched = true;
+    }
+    if (openChannel.node2_public_key === prevChannel.node1_public_key) {
+      openSide = 2;
+      prevLocal = 1;
+      prevRemote = 2;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    } else if (openChannel.node2_public_key === prevChannel.node2_public_key) {
+      openSide = 2;
+      prevLocal = 2;
+      prevRemote = 1;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    }
+
+    if (matched && !ambiguous) {
+      // fetch closing channel transaction and perform forensics on the outputs
+      let prevChannelTx = await this.fetchTransaction(input.txid, true);
+      let outspends: IEsploraApi.Outspend[] | undefined;
+      try {
+        outspends = await bitcoinApi.$getOutspends(input.txid);
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+      } catch (e) {
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+      }
+      if (!outspends || !prevChannelTx) {
+        return;
+      }
+      if (!linkedOpenings) {
+        if (!prevChannel.outputs || !prevChannel.outputs.length) {
+          prevChannel.outputs = prevChannelTx.vout.map(vout => {
+            return {
+              type: 0,
+              value: vout.value,
+            };
+          });
+        }
+        for (let i = 0; i < outspends?.length; i++) {
+          const outspend = outspends[i];
+          const output = prevChannel.outputs[i];
+          if (outspend.spent && outspend.txid) {
+            try {
+              const spendingTx = await this.fetchTransaction(outspend.txid, true);
+              if (spendingTx) {
+                output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              }
+            } catch (e) {
+              logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
+            }
+          } else {
+            output.type = 0;
+          }
+        }
+
+        // attribute outputs to each counterparty, and sum up total known balances
+        prevChannel.outputs[input.vout].node = prevLocal;
+        const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0;
+        const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type);
+        const mutualClose = ((prevChannel.status === 2 || prevChannel.status === 'closed') && prevChannel.closing_reason === 1);
+        let localClosingBalance = 0;
+        let remoteClosingBalance = 0;
+        for (const output of prevChannel.outputs) {
+          if (isPenalty) {
+            // penalty close, so local node takes everything
+            localClosingBalance += output.value;
+          } else if (output.node) {
+            // this output determinstically linked to one of the counterparties
+            if (output.node === prevLocal) {
+              localClosingBalance += output.value;
+            } else {
+              remoteClosingBalance += output.value;
+            }
+          } else if (normalOutput && (output.type === 1 || output.type === 3 || (mutualClose && prevChannel.outputs.length === 2))) {
+            // local node had one main output, therefore remote node takes the other
+            remoteClosingBalance += output.value;
+          }
+        }
+        prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance;
+        prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance;
+        prevChannel.closing_fee = prevChannelTx.fee;
+
+        if (initiator && !linkedOpenings) {
+          const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal;
+          prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`];
+        }
+  
+        // save changes to the closing channel
+        await channelsApi.$updateClosingInfo(prevChannel);
+      } else {
+        if (prevChannelTx.vin.length <= 1) {
+          prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity;
+          prevChannel.single_funded = true;
+          prevChannel.funding_ratio = 1;
+          // save changes to the closing channel
+          await channelsApi.$updateOpeningInfo(prevChannel);
+        }
+      }
+      openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0);
+    }
+  }
+
+  async fetchTransaction(txid: string, temp: boolean = false): Promise<IEsploraApi.Transaction | null> {
+    let tx = this.txCache[txid];
+    if (!tx) {
+      try {
+        tx = await bitcoinApi.$getRawTransaction(txid);
+        this.txCache[txid] = tx;
+        if (temp) {
+          this.tempCached.push(txid);
+        }
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+      } catch (e) {
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+        return null;
+      }
+    }
+    return tx;
+  }
+
+  clearTempCache(): void {
+    for (const txid of this.tempCached) {
+      delete this.txCache[txid];
+    }
+    this.tempCached = [];
+  }
+
+  truncateTempCache(): void {
+    if (this.tempCached.length > tempCacheSize) {
+      const removed = this.tempCached.splice(0, this.tempCached.length - tempCacheSize);
+      for (const txid of removed) {
+        delete this.txCache[txid];
+      }
+    }
+  }
+}
+
+export default new ForensicsService();

backend/src/tasks/lightning/network-sync.service.ts

@@ -0,0 +1,316 @@
+import DB from '../../database';
+import logger from '../../logger';
+import channelsApi from '../../api/explorer/channels.api';
+import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
+import config from '../../config';
+import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
+import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+import { $lookupNodeLocation } from './sync-tasks/node-locations';
+import lightningApi from '../../api/lightning/lightning-api-factory';
+import nodesApi from '../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
+import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
+import { Common } from '../../api/common';
+import blocks from '../../api/blocks';
+import NodeRecordsRepository from '../../repositories/NodeRecordsRepository';
+import forensicsService from './forensics.service';
+
+class NetworkSyncService {
+  loggerTimer = 0;
+  closedChannelsScanBlock = 0;
+
+  constructor() {}
+
+  public async $startService(): Promise<void> {
+    logger.info(`Starting lightning network sync service`, logger.tags.ln);
+
+    this.loggerTimer = new Date().getTime() / 1000;
+
+    await this.$runTasks();
+  }
+
+  private async $runTasks(): Promise<void> {
+    const taskStartTime = Date.now();
+    try {
+      logger.debug(`Updating nodes and channels`, logger.tags.ln);
+
+      const networkGraph = await lightningApi.$getNetworkGraph();
+      if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
+        logger.info(`LN Network graph is empty, retrying in 10 seconds`, logger.tags.ln);
+        setTimeout(() => { this.$runTasks(); }, 10000);
+        return;
+      }
+
+      await this.$updateNodesList(networkGraph.nodes);
+      await this.$updateChannelsList(networkGraph.edges);
+      await this.$deactivateChannelsWithoutActiveNodes();
+      await this.$lookUpCreationDateFromChain();
+      await this.$updateNodeFirstSeen();
+      await this.$scanForClosedChannels();
+      
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        // run forensics on new channels only
+        await forensicsService.$runClosedChannelsForensics(true);
+      }
+
+    } catch (e) {
+      logger.err(`$runTasks() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+    }
+
+    setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
+  }
+
+  /**
+   * Update the `nodes` table to reflect the current network graph state
+   */
+  private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
+    let progress = 0;
+
+    let deletedSockets = 0;
+    let deletedRecords = 0;
+    const graphNodesPubkeys: string[] = [];
+    for (const node of nodes) {
+      const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
+      node.last_update = Math.max(node.last_update ?? 0, latestUpdated);
+
+      await nodesApi.$saveNode(node);
+      graphNodesPubkeys.push(node.pub_key);
+      ++progress;
+
+      const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+      if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+        logger.debug(`Updating node ${progress}/${nodes.length}`, logger.tags.ln);
+        this.loggerTimer = new Date().getTime() / 1000;
+      }
+
+      const addresses: string[] = [];
+      for (const socket of node.addresses) {
+        await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket));
+        addresses.push(socket.addr);
+      }
+      deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+
+      const oldRecordTypes = await NodeRecordsRepository.$getRecordTypes(node.pub_key);
+      const customRecordTypes: number[] = [];
+      for (const [type, payload] of Object.entries(node.custom_records || {})) {
+        const numericalType = parseInt(type);
+        await NodeRecordsRepository.$saveRecord({
+          publicKey: node.pub_key,
+          type: numericalType,
+          payload,
+        });
+        customRecordTypes.push(numericalType);
+      }
+      if (oldRecordTypes.reduce((changed, type) => changed || customRecordTypes.indexOf(type) === -1, false)) {
+        deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes);
+      }
+    }
+    logger.debug(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);
+
+    // If a channel if not present in the graph, mark it as inactive
+    await nodesApi.$setNodesInactive(graphNodesPubkeys);
+
+    if (config.MAXMIND.ENABLED) {
+      $lookupNodeLocation();
+    }
+  }
+
+  /**
+   * Update the `channels` table to reflect the current network graph state
+   */
+  private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
+    try {
+      const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`);
+      const closedChannels = {};
+      for (const closedChannel of closedChannelsRaw) {
+        closedChannels[closedChannel.id] = true;
+      }
+
+      let progress = 0;
+
+      const graphChannelsIds: string[] = [];
+      for (const channel of channels) {
+        if (!closedChannels[channel.channel_id]) {
+          await channelsApi.$saveChannel(channel);
+        }
+        graphChannelsIds.push(channel.channel_id);
+        ++progress;
+
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+          logger.debug(`Updating channel ${progress}/${channels.length}`, logger.tags.ln);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      logger.debug(`${progress} channels updated`, logger.tags.ln);
+
+      // If a channel if not present in the graph, mark it as inactive
+      await channelsApi.$setChannelsInactive(graphChannelsIds);
+    } catch (e) {
+      logger.err(` Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.ln);
+    }
+  }
+
+  // This method look up the creation date of the earliest channel of the node
+  // and update the node to that date in order to get the earliest first seen date
+  private async $updateNodeFirstSeen(): Promise<void> {
+    let progress = 0;
+    let updated = 0;
+
+    try {
+      const [nodes]: any[] = await DB.query(`
+        SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node1_public_key = nodes.public_key
+        ) AS created1,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node2_public_key = nodes.public_key
+        ) AS created2
+        FROM nodes
+      `);
+
+      for (const node of nodes) {
+        const lowest = Math.min(
+          node.created1 ?? Number.MAX_SAFE_INTEGER,
+          node.created2 ?? Number.MAX_SAFE_INTEGER,
+          node.first_seen ?? Number.MAX_SAFE_INTEGER
+        );
+        if (lowest < node.first_seen) {
+          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
+          const params = [lowest, node.public_key];
+          ++updated;
+          await DB.query(query, params);
+        }
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+          logger.debug(`Updating node first seen date ${progress}/${nodes.length}`, logger.tags.ln);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      if (updated > 0) {
+        logger.debug(`Updated ${updated} node first seen dates`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err(`$updateNodeFirstSeen() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+    }
+  }
+
+  private async $lookUpCreationDateFromChain(): Promise<void> {
+    let progress = 0;
+
+    logger.debug(`Running channel creation date lookup`, logger.tags.ln);
+    try {
+      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
+      for (const channel of channels) {
+        const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
+        if (!transaction) {
+          continue;
+        }
+        await DB.query(`
+          UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
+          [transaction.timestamp, channel.id]
+        );
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+          logger.debug(`Updating channel creation date ${progress}/${channels.length}`, logger.tags.ln);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      if (channels.length > 0) {
+        logger.debug(`Updated ${channels.length} channels' creation date`, logger.tags.ln);
+      }      
+    } catch (e) {
+      logger.err(`$lookUpCreationDateFromChain() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+    }
+  }
+
+  /**
+   * If a channel does not have any active node linked to it, then also
+   * mark that channel as inactive
+   */
+  private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
+    logger.debug(`Find channels which nodes are offline`, logger.tags.ln);
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE channels.status = 1
+        AND (
+          (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node1_public_key
+            AND nodes.status = 1
+          ) = 0
+        OR (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node2_public_key
+            AND nodes.status = 1
+          ) = 0)
+        `);
+
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err(`$deactivateChannelsWithoutActiveNodes() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+    }
+  }
+
+  private async $scanForClosedChannels(): Promise<void> {
+    if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
+      logger.debug(`We've already scan closed channels for this block, skipping.`);
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      let log = `Starting closed channels scan`;
+      if (this.closedChannelsScanBlock > 0) {
+        log += `. Last scan was at block ${this.closedChannelsScanBlock}`;
+      } else {
+        log += ` for the first time`;
+      }
+      logger.info(`${log}`, logger.tags.ln);
+
+      const channels = await channelsApi.$getChannelsByStatus([0, 1]);
+      for (const channel of channels) {
+        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
+        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
+          logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln);
+          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
+            [spendingTx.status.block_time, channel.id]);
+          if (spendingTx.txid && !channel.closing_transaction_id) {
+            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+          }
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
+      logger.debug(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`, logger.tags.ln);
+    } catch (e) {
+      logger.err(`$scanForClosedChannels() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+    }
+  }
+}
+
+export default new NetworkSyncService();

backend/src/tasks/lightning/node-sync.service.ts

@@ -1,427 +0,0 @@
-import { chanNumber } from 'bolt07';
-import DB from '../../database';
-import logger from '../../logger';
-import channelsApi from '../../api/explorer/channels.api';
-import bitcoinClient from '../../api/bitcoin/bitcoin-client';
-import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
-import config from '../../config';
-import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
-import lightningApi from '../../api/lightning/lightning-api-factory';
-import { ILightningApi } from '../../api/lightning/lightning-api.interface';
-import { $lookupNodeLocation } from './sync-tasks/node-locations';
-
-class NodeSyncService {
-  constructor() {}
-
-  public async $startService() {
-    logger.info('Starting node sync service');
-
-    await this.$runUpdater();
-
-    setInterval(async () => {
-      await this.$runUpdater();
-    }, 1000 * 60 * 60);
-  }
-
-  private async $runUpdater() {
-    try {
-      logger.info(`Updating nodes and channels...`);
-
-      const networkGraph = await lightningApi.$getNetworkGraph();
-
-      for (const node of networkGraph.nodes) {
-        await this.$saveNode(node);
-      }
-      logger.info(`Nodes updated.`);
-
-      if (config.MAXMIND.ENABLED) {
-        await $lookupNodeLocation();
-      }
-
-      const graphChannelsIds: string[] = [];
-      for (const channel of networkGraph.channels) {
-        await this.$saveChannel(channel);
-        graphChannelsIds.push(channel.id);
-      }
-      await this.$setChannelsInactive(graphChannelsIds);
-
-      logger.info(`Channels updated.`);
-
-      await this.$findInactiveNodesAndChannels();
-      await this.$lookUpCreationDateFromChain();
-      await this.$updateNodeFirstSeen();
-      await this.$scanForClosedChannels();
-      if (config.MEMPOOL.BACKEND === 'esplora') {
-        await this.$runClosedChannelsForensics();
-      }
-
-    } catch (e) {
-      logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // This method look up the creation date of the earliest channel of the node
-  // and update the node to that date in order to get the earliest first seen date
-  private async $updateNodeFirstSeen() {
-    try {
-      const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
-      for (const node of nodes) {
-        let lowest = 0;
-        if (node.created1) {
-          if (node.created2 && node.created2 < node.created1) {
-            lowest = node.created2;
-          } else {
-            lowest = node.created1;
-          }
-        } else if (node.created2) {
-          lowest = node.created2;
-        }
-        if (lowest && lowest < node.first_seen) {
-          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
-          const params = [lowest, node.public_key];
-          await DB.query(query, params);
-        }
-      }
-      logger.info(`Node first seen dates scan complete.`);
-    } catch (e) {
-      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $lookUpCreationDateFromChain() {
-    logger.info(`Running channel creation date lookup...`);
-    try {
-      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
-      for (const channel of channels) {
-        const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
-        await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
-      }
-      logger.info(`Channel creation dates scan complete.`);
-    } catch (e) {
-      logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // Looking for channels whos nodes are inactive
-  private async $findInactiveNodesAndChannels(): Promise<void> {
-    logger.info(`Running inactive channels scan...`);
-
-    try {
-      // @ts-ignore
-      const [channels]: [ILightningApi.Channel[]] = await DB.query(`
-        SELECT channels.id
-        FROM channels
-        WHERE channels.status = 1
-        AND (
-          (
-            SELECT COUNT(*)
-            FROM nodes
-            WHERE nodes.public_key = channels.node1_public_key
-          ) = 0
-        OR (
-            SELECT COUNT(*)
-            FROM nodes
-            WHERE nodes.public_key = channels.node2_public_key
-          ) = 0)
-        `);
-
-      for (const channel of channels) {
-        await this.$updateChannelStatus(channel.id, 0);
-      }
-      logger.info(`Inactive channels scan complete.`);
-    } catch (e) {
-      logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $scanForClosedChannels(): Promise<void> {
-    try {
-      logger.info(`Starting closed channels scan...`);
-      const channels = await channelsApi.$getChannelsByStatus(0);
-      for (const channel of channels) {
-        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
-        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
-          logger.debug('Marking channel: ' + channel.id + ' as closed.');
-          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
-            [spendingTx.status.block_time, channel.id]);
-          if (spendingTx.txid && !channel.closing_transaction_id) {
-            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
-          }
-        }
-      }
-      logger.info(`Closed channels scan complete.`);
-    } catch (e) {
-      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  /*
-    1. Mutually closed
-    2. Forced closed
-    3. Forced closed with penalty
-  */
-
-  private async $runClosedChannelsForensics(): Promise<void> {
-    if (!config.ESPLORA.REST_API_URL) {
-      return;
-    }
-    try {
-      logger.info(`Started running closed channel forensics...`);
-      const channels = await channelsApi.$getClosedChannelsWithoutReason();
-      for (const channel of channels) {
-        let reason = 0;
-        // Only Esplora backend can retrieve spent transaction outputs
-        const outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
-        const lightningScriptReasons: number[] = [];
-        for (const outspend of outspends) {
-          if (outspend.spent && outspend.txid) {
-            const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
-            const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
-            lightningScriptReasons.push(lightningScript);
-          }
-        }
-        if (lightningScriptReasons.length === outspends.length
-          && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
-          reason = 1;
-        } else {
-          const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
-          if (filteredReasons.length) {
-            if (filteredReasons.some((r) => r === 2 || r === 4)) {
-              reason = 3;
-            } else {
-              reason = 2;
-            }
-          } else {
-            /*
-              We can detect a commitment transaction (force close) by reading Sequence and Locktime
-              https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
-            */
-            const closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
-            const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
-            const locktimeHex: string = closingTx.locktime.toString(16);
-            if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
-              reason = 2; // Here we can't be sure if it's a penalty or not
-            } else {
-              reason = 1;
-            }
-          }
-        }
-        if (reason) {
-          logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
-          await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
-        }
-      }
-      logger.info(`Closed channels forensics scan complete.`);
-    } catch (e) {
-      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private findLightningScript(vin: IEsploraApi.Vin): number {
-    const topElement = vin.witness[vin.witness.length - 2];
-      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
-        if (topElement === '01') {
-          // top element is '01' to get in the revocation path
-          // 'Revoked Lightning Force Close';
-          // Penalty force closed
-          return 2;
-        } else {
-          // top element is '', this is a delayed to_local output
-          // 'Lightning Force Close';
-          return 3;
-        }
-      } else if (
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
-      ) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
-        if (topElement.length === 66) {
-          // top element is a public key
-          // 'Revoked Lightning HTLC'; Penalty force closed
-          return 4;
-        } else if (topElement) {
-          // top element is a preimage
-          // 'Lightning HTLC';
-          return 5;
-        } else {
-          // top element is '' to get in the expiry of the script
-          // 'Expired Lightning HTLC';
-          return 6;
-        }
-      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
-        if (topElement) {
-          // top element is a signature
-          // 'Lightning Anchor';
-          return 7;
-        } else {
-          // top element is '', it has been swept after 16 blocks
-          // 'Swept Lightning Anchor';
-          return 8;
-        }
-      }
-      return 1;
-  }
-
-  private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
-    const fromChannel = chanNumber({ channel: channel.id }).number;
-
-    try {
-      const query = `INSERT INTO channels
-        (
-          id,
-          short_id,
-          capacity,
-          transaction_id,
-          transaction_vout,
-          updated_at,
-          status,
-          node1_public_key,
-          node1_base_fee_mtokens,
-          node1_cltv_delta,
-          node1_fee_rate,
-          node1_is_disabled,
-          node1_max_htlc_mtokens,
-          node1_min_htlc_mtokens,
-          node1_updated_at,
-          node2_public_key,
-          node2_base_fee_mtokens,
-          node2_cltv_delta,
-          node2_fee_rate,
-          node2_is_disabled,
-          node2_max_htlc_mtokens,
-          node2_min_htlc_mtokens,
-          node2_updated_at
-        )
-        VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        ON DUPLICATE KEY UPDATE
-          capacity = ?,
-          updated_at = ?,
-          status = 1,
-          node1_public_key = ?,
-          node1_base_fee_mtokens = ?,
-          node1_cltv_delta = ?,
-          node1_fee_rate = ?,
-          node1_is_disabled = ?,
-          node1_max_htlc_mtokens = ?,
-          node1_min_htlc_mtokens = ?,
-          node1_updated_at = ?,
-          node2_public_key = ?,
-          node2_base_fee_mtokens = ?,
-          node2_cltv_delta = ?,
-          node2_fee_rate = ?,
-          node2_is_disabled = ?,
-          node2_max_htlc_mtokens = ?,
-          node2_min_htlc_mtokens = ?,
-          node2_updated_at = ?
-        ;`;
-
-      await DB.query(query, [
-        fromChannel,
-        channel.id,
-        channel.capacity,
-        channel.transaction_id,
-        channel.transaction_vout,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-        channel.capacity,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-      ]);
-    } catch (e) {
-      logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> {
-    try {
-      await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]);
-    } catch (e) {
-      logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
-    try {
-      await DB.query(`
-        UPDATE channels
-        SET status = 0
-        WHERE short_id NOT IN (
-          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
-        )
-        AND status != 2
-      `);
-    } catch (e) {
-      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $saveNode(node: ILightningApi.Node): Promise<void> {
-    try {
-      const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00';
-      const sockets = node.sockets.join(',');
-      const query = `INSERT INTO nodes(
-          public_key,
-          first_seen,
-          updated_at,
-          alias,
-          color,
-          sockets
-        )
-        VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
-
-      await DB.query(query, [
-        node.public_key,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-      ]);
-    } catch (e) {
-      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private utcDateToMysql(dateString: string): string {
-    const d = new Date(Date.parse(dateString));
-    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
-  }
-}
-
-export default new NodeSyncService();

backend/src/tasks/lightning/stats-updater.service.ts

@@ -1,350 +1,33 @@
-
-import DB from '../../database';
 import logger from '../../logger';
 import lightningApi from '../../api/lightning/lightning-api-factory';
-import channelsApi from '../../api/explorer/channels.api';
-import * as net from 'net';
+import LightningStatsImporter from './sync-tasks/stats-importer';
+import config from '../../config';
+import { Common } from '../../api/common';
 
 class LightningStatsUpdater {
-  hardCodedStartTime = '2018-01-12';
+  public async $startService(): Promise<void> {
+    logger.info(`Starting Lightning Stats service`, logger.tags.ln);
 
-  public async $startService() {
-    logger.info('Starting Lightning Stats service');
-    let isInSync = false;
-    let error: any;
-    try {
-      error = null;
-      isInSync = await this.$lightningIsSynced();
-    } catch (e) {
-      error = e;
-    }
-    if (!isInSync) {
-      if (error) {
-        logger.warn('Was not able to fetch Lightning Node status: ' + (error instanceof Error ? error.message : error) + '. Retrying in 1 minute...');
-      } else {
-        logger.notice('The Lightning graph is not yet in sync. Retrying in 1 minute...');
-      }
-      setTimeout(() => this.$startService(), 60 * 1000);
-      return;
-    }
-
-    await this.$populateHistoricalStatistics();
-    await this.$populateHistoricalNodeStatistics();
-
-    setTimeout(() => {
-      this.$runTasks();
-    }, this.timeUntilMidnight());
-  }
-
-  private timeUntilMidnight(): number {
-    const date = new Date();
-    this.setDateMidnight(date);
-    date.setUTCHours(24);
-    return date.getTime() - new Date().getTime();
-  }
-
-  private setDateMidnight(date: Date): void {
-    date.setUTCHours(0);
-    date.setUTCMinutes(0);
-    date.setUTCSeconds(0);
-    date.setUTCMilliseconds(0);
-  }
-
-  private async $lightningIsSynced(): Promise<boolean> {
-    const nodeInfo = await lightningApi.$getInfo();
-    return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph;
+    await this.$runTasks();
+    LightningStatsImporter.$run();
   }
 
   private async $runTasks(): Promise<void> {
-    await this.$logLightningStatsDaily();
-    await this.$logNodeStatsDaily();
+    await this.$logStatsDaily();
 
-    setTimeout(() => {
-      this.$runTasks();
-    }, this.timeUntilMidnight());
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
   }
 
-  private async $logLightningStatsDaily() {
-    try {
-      logger.info(`Running lightning daily stats log...`);  
-
-      const networkGraph = await lightningApi.$getNetworkGraph();
-      let total_capacity = 0;
-      for (const channel of networkGraph.channels) {
-        if (channel.capacity) {
-          total_capacity += channel.capacity;
-        }
-      }
-
-      let clearnetNodes = 0;
-      let torNodes = 0;
-      let unannouncedNodes = 0;
-      for (const node of networkGraph.nodes) {
-        let isUnnanounced = true;
-        for (const socket of node.sockets) {
-          const hasOnion = socket.indexOf('.onion') !== -1;
-          if (hasOnion) {
-            torNodes++;
-            isUnnanounced = false;
-          }
-          const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
-          if (hasClearnet) {
-            clearnetNodes++;
-            isUnnanounced = false;
-          }
-        }
-        if (isUnnanounced) {
-          unannouncedNodes++;
-        }
-      }
-
-      const channelStats = await channelsApi.$getChannelsStats();
-
-      const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes,
-          avg_capacity,
-          avg_fee_rate,
-          avg_base_fee_mtokens,
-          med_capacity,
-          med_fee_rate,
-          med_base_fee_mtokens
-        )
-        VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-      await DB.query(query, [
-        networkGraph.channels.length,
-        networkGraph.nodes.length,
-        total_capacity,
-        torNodes,
-        clearnetNodes,
-        unannouncedNodes,
-        channelStats.avgCapacity,
-        channelStats.avgFeeRate,
-        channelStats.avgBaseFee,
-        channelStats.medianCapacity,
-        channelStats.medianFeeRate,
-        channelStats.medianBaseFee,
-      ]);
-      logger.info(`Lightning daily stats done.`);
-    } catch (e) {
-      logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $logNodeStatsDaily() {
-    try {
-      logger.info(`Running daily node stats update...`);
-
-      const query = `
-        SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left,
-          c2.channels_capacity_right
-        FROM nodes
-        LEFT JOIN (
-          SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left
-          FROM channels
-          WHERE channels.status = 1
-          GROUP BY node1_public_key
-        ) c1 ON c1.node1_public_key = nodes.public_key
-        LEFT JOIN (
-          SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right
-          FROM channels WHERE channels.status = 1 GROUP BY node2_public_key
-        ) c2 ON c2.node2_public_key = nodes.public_key
-      `;
-      
-      const [nodes]: any = await DB.query(query);
-
-      for (const node of nodes) {
-        await DB.query(
-          `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`,
-          [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
-            node.channels_count_left + node.channels_count_right]);
-      }
-      logger.info('Daily node stats has updated.');
-    } catch (e) {
-      logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // We only run this on first launch
-  private async $populateHistoricalStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical stats population...`);
-
-      const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`);
-      const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`);
-
-      const date: Date = new Date(this.hardCodedStartTime);
-      const currentDate = new Date();
-      this.setDateMidnight(currentDate);
-
-      while (date < currentDate) {
-        let totalCapacity = 0;
-        let channelsCount = 0;
-
-        for (const channel of channels) {
-          if (new Date(channel.created) > date) {
-            break;
-          }
-          if (channel.closing_date === null || new Date(channel.closing_date) > date) {
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-        }
-
-        let nodeCount = 0;
-        let clearnetNodes = 0;
-        let torNodes = 0;
-        let unannouncedNodes = 0;
-
-        for (const node of nodes) {
-          if (new Date(node.first_seen) > date) {
-            break;
-          }
-          nodeCount++;
-
-          const sockets = node.sockets.split(',');
-          let isUnnanounced = true;
-          for (const socket of sockets) {
-            const hasOnion = socket.indexOf('.onion') !== -1;
-            if (hasOnion) {
-              torNodes++;
-              isUnnanounced = false;
-            }
-            const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':'))));
-            if (hasClearnet) {
-              clearnetNodes++;
-              isUnnanounced = false;
-            }
-          }
-          if (isUnnanounced) {
-            unannouncedNodes++;
-          }
-        }
-
-        const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes,
-          avg_capacity,
-          avg_fee_rate,
-          avg_base_fee_mtokens,
-          med_capacity,
-          med_fee_rate,
-          med_base_fee_mtokens
-        )
-        VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-        const rowTimestamp = date.getTime() / 1000; // Save timestamp for the row insertion down below
-
-        date.setUTCDate(date.getUTCDate() + 1);
-
-        // Last iteration, save channels stats
-        const channelStats = (date >= currentDate ? await channelsApi.$getChannelsStats() : undefined);
-
-        await DB.query(query, [
-          rowTimestamp,
-          channelsCount,
-          nodeCount,
-          totalCapacity,
-          torNodes,
-          clearnetNodes,
-          unannouncedNodes,
-          channelStats?.avgCapacity ?? 0,
-          channelStats?.avgFeeRate ?? 0,
-          channelStats?.avgBaseFee ?? 0,
-          channelStats?.medianCapacity ?? 0,
-          channelStats?.medianFeeRate ?? 0,
-          channelStats?.medianBaseFee ?? 0,
-          ]);
-      }
-
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $populateHistoricalNodeStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical node stats population...`);
-
-      const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`);
-
-      for (const node of nodes) {
-        const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]);
-        
-        const date: Date = new Date(this.hardCodedStartTime);
-        const currentDate = new Date();
-        this.setDateMidnight(currentDate);
-
-        let lastTotalCapacity = 0;
-        let lastChannelsCount = 0;
-
-        while (date < currentDate) {
-          let totalCapacity = 0;
-          let channelsCount = 0;
-          for (const channel of channels) {
-            if (new Date(channel.created) > date) {
-              break;
-            }
-            if (channel.closing_date !== null && new Date(channel.closing_date) < date) {
-              date.setUTCDate(date.getUTCDate() + 1);
-              continue;
-            }
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-
-          if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) {
-            date.setUTCDate(date.getUTCDate() + 1);
-            continue;
-          }
-
-          lastTotalCapacity = totalCapacity;
-          lastChannelsCount = channelsCount;
-  
-          const query = `INSERT INTO node_stats(
-            public_key,
-            added,
-            capacity,
-            channels
-          )
-          VALUES (?, FROM_UNIXTIME(?), ?, ?)`;
-
-          await DB.query(query, [
-            node.public_key,
-            date.getTime() / 1000,
-            totalCapacity,
-            channelsCount,
-          ]);
-          date.setUTCDate(date.getUTCDate() + 1);
-        }
-        logger.debug('Updated node_stats for: ' + node.alias);
-      }
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e));
-    }
+  /**
+   * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
+   */
+  private async $logStatsDaily(): Promise<void> {
+    const date = new Date();
+    Common.setDateMidnight(date);
+    const networkGraph = await lightningApi.$getNetworkGraph();
+    await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
+    
+    logger.debug(`Updated latest network stats`, logger.tags.ln);
   }
 }
 

backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts

@@ -0,0 +1,122 @@
+import { existsSync, promises } from 'fs';
+import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
+import { Common } from '../../../api/common';
+import config from '../../../config';
+import logger from '../../../logger';
+
+const fsPromises = promises;
+
+const BLOCKS_CACHE_MAX_SIZE = 100;  
+const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json';
+
+class FundingTxFetcher {
+  private running = false;
+  private blocksCache = {};
+  private channelNewlyProcessed = 0;
+  public fundingTxCache = {};
+
+  async $init(): Promise<void> {
+    // Load funding tx disk cache
+    if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) {
+      try {
+        this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
+      } catch (e) {
+        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`, logger.tags.ln);
+        this.fundingTxCache = {};
+      }
+      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`, logger.tags.ln);
+    }
+  }
+
+  async $fetchChannelsFundingTxs(channelIds: string[]): Promise<void> {
+    if (this.running) {
+      return;
+    }
+    this.running = true;
+    
+    const globalTimer = new Date().getTime() / 1000;
+    let cacheTimer = new Date().getTime() / 1000;
+    let loggerTimer = new Date().getTime() / 1000;
+    let channelProcessed = 0;
+    this.channelNewlyProcessed = 0;
+    for (const channelId of channelIds) {
+      await this.$fetchChannelOpenTx(channelId);
+      ++channelProcessed;
+
+      let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+      if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+        elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
+        logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
+          `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
+          `elapsed: ${elapsedSeconds} seconds`,
+          logger.tags.ln
+        );
+        loggerTimer = new Date().getTime() / 1000;
+      }
+
+      elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
+      if (elapsedSeconds > 60) {
+        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
+        fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+        cacheTimer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (this.channelNewlyProcessed > 0) {
+      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`, logger.tags.ln);
+      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
+      fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+    }
+
+    this.running = false;
+  }
+  
+  public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number} | null> {
+    channelId = Common.channelIntegerIdToShortId(channelId);
+
+    if (this.fundingTxCache[channelId]) {
+      return this.fundingTxCache[channelId];
+    }
+
+    const parts = channelId.split('x');
+    const blockHeight = parts[0];
+    const txIdx = parts[1];
+    const outputIdx = parts[2];
+
+    let block = this.blocksCache[blockHeight];
+    // Fetch it from core
+    if (!block) {
+      const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10));
+      block = await bitcoinClient.getBlock(blockHash, 1);
+    }
+    this.blocksCache[block.height] = block;
+
+    const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse();
+    if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) {
+      for (let i = 0; i < 10; ++i) {
+        delete this.blocksCache[blocksCacheHashes[i]];
+      }
+    }
+
+    const txid = block.tx[txIdx];
+    const rawTx = await bitcoinClient.getRawTransaction(txid);
+    const tx = await bitcoinClient.decodeRawTransaction(rawTx);
+
+    if (!tx || !tx.vout || tx.vout.length < parseInt(outputIdx, 10) + 1 || tx.vout[outputIdx].value === undefined) {
+      logger.err(`Cannot find blockchain funding tx for channel id ${channelId}. Possible reasons are: bitcoin backend timeout or the channel shortId is not valid`);
+      return null;
+    }
+
+    this.fundingTxCache[channelId] = {
+      timestamp: block.time,
+      txid: txid,
+      value: tx.vout[outputIdx].value,
+    };
+
+    ++this.channelNewlyProcessed;
+
+    return this.fundingTxCache[channelId];
+  }
+}
+
+export default new FundingTxFetcher;

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -4,38 +4,75 @@ import nodesApi from '../../../api/explorer/nodes.api';
 import config from '../../../config';
 import DB from '../../../database';
 import logger from '../../../logger';
+import { ResultSetHeader } from 'mysql2';
+import * as IPCheck from '../../../utils/ipcheck.js';
+import { Reader } from 'mmdb-lib';
 
 export async function $lookupNodeLocation(): Promise<void> {
-  logger.info(`Running node location updater using Maxmind...`);
+  let loggerTimer = new Date().getTime() / 1000;
+  let progress = 0;
+  let nodesUpdated = 0;
+  let geoNamesInserted = 0;
+
+  logger.debug(`Running node location updater using Maxmind`, logger.tags.ln);
   try {
     const nodes = await nodesApi.$getAllNodes();
     const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
     const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
-    const lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
+    let lookupIsp: Reader<IspResponse> | null = null;
+    try {
+      lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
+    } catch (e) { }
 
     for (const node of nodes) {
       const sockets: string[] = node.sockets.split(',');
       for (const socket of sockets) {
         const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
         const hasClearnet = [4, 6].includes(net.isIP(ip));
+
         if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
           const city = lookupCity.get(ip);
           const asn = lookupAsn.get(ip);
-          const isp = lookupIsp.get(ip);
+          let isp: IspResponse | null = null;
+          if (lookupIsp) {
+            isp = lookupIsp.get(ip);
+          }
+
+          let asOverwrite: any | undefined;
+          if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
+            asOverwrite = {
+              asn: 394745,
+              name: 'Lunanode',
+            };
+          }
+          else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) {
+            asOverwrite = {
+              asn: 30058,
+              name: 'FDCservers.net',
+            };
+          }
+          else if (asn && asn.autonomous_system_number === 174) {
+            asOverwrite = {
+              asn: 174,
+              name: 'Cogent Communications',
+            };
+          }
 
           if (city && (asn || isp)) {
-            const query = `UPDATE nodes SET 
-              as_number = ?, 
-              city_id = ?, 
-              country_id = ?, 
-              subdivision_id = ?, 
-              longitude = ?, 
-              latitude = ?, 
-              accuracy_radius = ?
-            WHERE public_key = ?`;
+            const query = `
+              UPDATE nodes SET 
+                as_number = ?, 
+                city_id = ?, 
+                country_id = ?, 
+                subdivision_id = ?, 
+                longitude = ?, 
+                latitude = ?, 
+                accuracy_radius = ?
+              WHERE public_key = ?
+            `;
 
             const params = [
-              isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+              asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
               city.city?.geoname_id,
               city.country?.geoname_id,
               city.subdivisions ? city.subdivisions[0].geoname_id : null,
@@ -44,54 +81,88 @@ export async function $lookupNodeLocation(): Promise<void> {
               city.location?.accuracy_radius,
               node.public_key
             ];
-            await DB.query(query, params);
+            let result = await DB.query<ResultSetHeader>(query, params);
+            if (result[0].changedRows ?? 0 > 0) {
+              ++nodesUpdated;
+            }
 
-             // Store Continent
-             if (city.continent?.geoname_id) {
-               await DB.query(
+            // Store Continent
+            if (city.continent?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
                 [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
-             }
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
 
-             // Store Country
-             if (city.country?.geoname_id) {
-               await DB.query(
+            // Store Country
+            if (city.country?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
                 [city.country?.geoname_id, JSON.stringify(city.country?.names)]);
-             }
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
 
             // Store Country ISO code
             if (city.country?.iso_code) {
-              await DB.query(
-               `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
-               [city.country?.geoname_id, city.country?.iso_code]);
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
+                [city.country?.geoname_id, city.country?.iso_code]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store Division
             if (city.subdivisions && city.subdivisions[0]) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
                 [city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store City
             if (city.city?.geoname_id) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
                 [city.city?.geoname_id, JSON.stringify(city.city?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store AS name
             if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
-                [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
+                [
+                  asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+                  JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
+                ]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
           }
+
+          ++progress;
+          const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+          if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+            logger.debug(`Updating node location data ${progress}/${nodes.length}`);
+            loggerTimer = new Date().getTime() / 1000;
+          }
         }
       }
     }
-    logger.info(`Node location data updated.`);
+
+    if (nodesUpdated > 0) {
+      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`, logger.tags.ln);
+    }
   } catch (e) {
     logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
   }

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -0,0 +1,546 @@
+import DB from '../../../database';
+import { promises } from 'fs';
+import logger from '../../../logger';
+import fundingTxFetcher from './funding-tx-fetcher';
+import config from '../../../config';
+import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
+import { isIP } from 'net';
+import { Common } from '../../../api/common';
+import channelsApi from '../../../api/explorer/channels.api';
+import nodesApi from '../../../api/explorer/nodes.api';
+
+const fsPromises = promises;
+
+class LightningStatsImporter {
+  topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
+
+  async $run(): Promise<void> {
+    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+    logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
+    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+
+    if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
+      return;
+    }
+
+    await this.$importHistoricalLightningStats();
+    await this.$cleanupIncorrectSnapshot();
+  }
+
+  /**
+   * Generate LN network stats for one day
+   */
+  public async computeNetworkStats(timestamp: number,
+    networkGraph: ILightningApi.NetworkGraph, isHistorical: boolean = false): Promise<unknown> {
+    // Node counts and network shares
+    let clearnetNodes = 0;
+    let torNodes = 0;
+    let clearnetTorNodes = 0;
+    let unannouncedNodes = 0;
+
+    const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`);
+    const nodesInDb = {};
+    for (const node of nodesInDbRaw) {
+      nodesInDb[node.public_key] = node;
+    }
+
+    for (const node of networkGraph.nodes) {
+      // If we don't know about this node, insert it in db
+      if (isHistorical === true && !nodesInDb[node.pub_key]) {
+        await nodesApi.$saveNode({
+          last_update: node.last_update,
+          pub_key: node.pub_key,
+          alias: node.alias,
+          addresses: node.addresses,
+          color: node.color,
+          features: node.features,
+        });
+        nodesInDb[node.pub_key] = node;
+      } else {
+        await nodesApi.$updateNodeSockets(node.pub_key, node.addresses);
+      }
+
+      let hasOnion = false;
+      let hasClearnet = false;
+      let isUnnanounced = true;
+
+      for (const socket of (node.addresses ?? [])) {
+        if (!socket.network?.length && !socket.addr?.length) {
+          continue;
+        }
+        hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1 || socket.addr.indexOf('torv2') !== -1 || socket.addr.indexOf('torv3') !== -1;
+        hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])) || socket.addr.indexOf('ipv4') !== -1 || socket.addr.indexOf('ipv6') !== -1;;
+      }
+      if (hasOnion && hasClearnet) {
+        clearnetTorNodes++;
+        isUnnanounced = false;
+      } else if (hasOnion) {
+        torNodes++;
+        isUnnanounced = false;
+      } else if (hasClearnet) {
+        clearnetNodes++;
+        isUnnanounced = false;
+      }
+      if (isUnnanounced) {
+        unannouncedNodes++;
+      }
+    }
+
+    // Channels and node historical stats
+    const nodeStats = {};
+    let capacity = 0;
+    let avgFeeRate = 0;
+    let avgBaseFee = 0;
+    const capacities: number[] = [];
+    const feeRates: number[] = [];
+    const baseFees: number[] = [];
+    const alreadyCountedChannels = {};
+    
+    const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
+    const channelsInDb = {};
+    for (const channel of channelsInDbRaw) {
+      channelsInDb[channel.short_id] = channel;
+    }
+
+    for (const channel of networkGraph.edges) {
+      const short_id = Common.channelIntegerIdToShortId(channel.channel_id);
+
+      const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
+      if (!tx) {
+        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`, logger.tags.ln);
+        continue;
+      }
+
+      // If we don't know about this channel, insert it in db
+      if (isHistorical === true && !channelsInDb[short_id]) {
+        await channelsApi.$saveChannel({
+          channel_id: short_id,
+          chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
+          last_update: channel.last_update,
+          node1_pub: channel.node1_pub,
+          node2_pub: channel.node2_pub,
+          capacity: (tx.value * 100000000).toString(),
+          node1_policy: null,
+          node2_policy: null,
+        }, 0);
+        channelsInDb[channel.channel_id] = channel;
+      }
+
+      if (!nodeStats[channel.node1_pub]) {
+        nodeStats[channel.node1_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      if (!nodeStats[channel.node2_pub]) {
+        nodeStats[channel.node2_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      
+      if (!alreadyCountedChannels[short_id]) {
+        capacity += Math.round(tx.value * 100000000);
+        capacities.push(Math.round(tx.value * 100000000));
+        alreadyCountedChannels[short_id] = true;
+
+        nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node1_pub].channels++;
+        nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node2_pub].channels++;
+      }
+
+      if (isHistorical === false) { // Coming from the node
+        for (const policy of [channel.node1_policy, channel.node2_policy]) {
+          if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) {
+            avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
+            feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
+          }  
+          if (policy && parseInt(policy.fee_base_msat, 10) < 5000) {
+            avgBaseFee += parseInt(policy.fee_base_msat, 10);
+            baseFees.push(parseInt(policy.fee_base_msat, 10));
+          }
+        }
+      } else {
+        // @ts-ignore
+        if (channel.node1_policy.fee_rate_milli_msat < 5000) {
+          // @ts-ignore
+          avgFeeRate += parseInt(channel.node1_policy.fee_rate_milli_msat, 10);
+          // @ts-ignore
+          feeRates.push(parseInt(channel.node1_policy.fee_rate_milli_msat), 10);
+        }
+        // @ts-ignore
+        if (channel.node1_policy.fee_base_msat < 5000) {
+          // @ts-ignore
+          avgBaseFee += parseInt(channel.node1_policy.fee_base_msat, 10);
+          // @ts-ignore
+          baseFees.push(parseInt(channel.node1_policy.fee_base_msat), 10);
+        }
+      }
+    }
+
+    let medCapacity = 0;
+    let medFeeRate = 0;
+    let medBaseFee = 0;
+    let avgCapacity = 0;
+
+    avgFeeRate /= Math.max(networkGraph.edges.length, 1);
+    avgBaseFee /= Math.max(networkGraph.edges.length, 1);
+
+    if (capacities.length > 0) {
+      medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
+      avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
+    }
+    if (feeRates.length > 0) {
+      medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
+    }
+    if (baseFees.length > 0) {
+      medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
+    }
+
+    let query = `INSERT INTO lightning_stats(
+        added,
+        channel_count,
+        node_count,
+        total_capacity,
+        tor_nodes,
+        clearnet_nodes,
+        unannounced_nodes,
+        clearnet_tor_nodes,
+        avg_capacity,
+        avg_fee_rate,
+        avg_base_fee_mtokens,
+        med_capacity,
+        med_fee_rate,
+        med_base_fee_mtokens
+      )
+      VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+      added = FROM_UNIXTIME(?),
+      channel_count = ?,
+      node_count = ?,
+      total_capacity = ?,
+      tor_nodes = ?,
+      clearnet_nodes = ?,
+      unannounced_nodes = ?,
+      clearnet_tor_nodes = ?,
+      avg_capacity = ?,
+      avg_fee_rate = ?,
+      avg_base_fee_mtokens = ?,
+      med_capacity = ?,
+      med_fee_rate = ?,
+      med_base_fee_mtokens = ?
+    `;
+
+    await DB.query(query, [
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+    ]);
+
+    for (const public_key of Object.keys(nodeStats)) {
+      query = `INSERT INTO node_stats(
+          public_key,
+          added,
+          capacity,
+          channels
+        )
+        VALUES (?, FROM_UNIXTIME(?), ?, ?)
+        ON DUPLICATE KEY UPDATE
+        added = FROM_UNIXTIME(?),
+        capacity = ?,
+        channels = ?
+      `;
+
+      await DB.query(query, [
+        public_key,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+      ]);
+
+      if (!isHistorical) {
+        await DB.query(
+          `UPDATE nodes SET capacity = ?, channels = ? WHERE public_key = ?`,
+          [
+            nodeStats[public_key].capacity,
+            nodeStats[public_key].channels,
+            public_key,
+          ]
+        );
+      }
+    }
+
+    return {
+      added: timestamp,
+      node_count: networkGraph.nodes.length
+    };
+  }
+
+  /**
+   * Import topology files LN historical data into the database
+   */
+  async $importHistoricalLightningStats(): Promise<void> {
+    if (!config.LIGHTNING.TOPOLOGY_FOLDER) {
+      logger.info(`Lightning topology folder is not set. Not importing historical LN stats`);
+      return;
+    }
+
+    logger.debug('Run the historical importer');
+    try {
+      let fileList: string[] = [];
+      try {
+        fileList = await fsPromises.readdir(this.topologiesFolder);
+      } catch (e) {
+        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`, logger.tags.ln);
+        throw e;
+      }
+      // Insert history from the most recent to the oldest
+      // This also put the .json cached files first
+      fileList.sort().reverse();
+
+      const [rows]: any[] = await DB.query(`
+        SELECT UNIX_TIMESTAMP(added) AS added
+        FROM lightning_stats
+        ORDER BY added DESC
+      `);
+      const existingStatsTimestamps = {};
+      for (const row of rows) {
+        existingStatsTimestamps[row.added] = row;
+      }
+
+      // For logging purpose
+      let processed = 10;
+      let totalProcessed = 0;
+      let logStarted = false;
+
+      for (const filename of fileList) {
+        processed++;
+
+        const timestamp = parseInt(filename.split('_')[1], 10);
+
+        // Stats exist already, don't calculate/insert them
+        if (existingStatsTimestamps[timestamp] !== undefined) {
+          totalProcessed++;
+          continue;
+        }
+
+        if (filename.indexOf('topology_') === -1) {
+          totalProcessed++;
+          continue;
+        }
+
+        logger.debug(`Reading ${this.topologiesFolder}/${filename}`, logger.tags.ln);
+        let fileContent = '';
+        try {
+          fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
+        } catch (e: any) {
+          if (e.errno == -1) { // EISDIR - Ignore directorie
+            totalProcessed++;
+            continue;
+          }
+          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`, logger.tags.ln);
+          totalProcessed++;
+          continue;
+        }
+
+        let graph;
+        try {
+          graph = JSON.parse(fileContent);
+          graph = await this.cleanupTopology(graph);
+        } catch (e) {
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+          totalProcessed++;
+          continue;
+        }
+    
+        if (this.isIncorrectSnapshot(timestamp, graph)) {
+          logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
+          ++totalProcessed;
+          continue;
+        }
+
+        if (!logStarted) {
+          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`, logger.tags.ln);
+          logStarted = true;
+        }
+        
+        const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
+        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`, logger.tags.ln);
+
+        totalProcessed++;
+
+        if (processed > 10) {
+          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`, logger.tags.ln);
+          processed = 0;
+        } else {
+          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`, logger.tags.ln);
+        }
+        await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
+        const stat = await this.computeNetworkStats(timestamp, graph, true);
+
+        existingStatsTimestamps[timestamp] = stat;
+      }
+
+      if (totalProcessed > 0) {
+        logger.info(`Lightning network stats historical import completed`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+    }
+  }
+
+  cleanupTopology(graph): ILightningApi.NetworkGraph {
+    const newGraph = {
+      nodes: <ILightningApi.Node[]>[],
+      edges: <ILightningApi.Channel[]>[],
+    };
+
+    for (const node of graph.nodes) {
+      const addressesParts = (node.addresses ?? '').split(',');
+      const addresses: any[] = [];
+      for (const address of addressesParts) {
+        const formatted = Common.findSocketNetwork(address);
+        addresses.push({
+          network: formatted.network,
+          addr: formatted.url
+        });
+      }
+
+      let rgb = node.rgb_color ?? '#000000';
+      if (rgb.indexOf('#') === -1) {
+        rgb = `#${rgb}`;
+      }
+      newGraph.nodes.push({
+        last_update: node.timestamp ?? 0,
+        pub_key: node.id ?? null,
+        alias: node.alias ?? node.id.slice(0, 20),
+        addresses: addresses,
+        color: rgb,
+        features: {},
+      });
+    }
+
+    for (const adjacency of graph.adjacency) {
+      if (adjacency.length === 0) {
+        continue;
+      } else {
+        for (const edge of adjacency) {
+          newGraph.edges.push({
+            channel_id: edge.scid,
+            chan_point: '',
+            last_update: edge.timestamp,
+            node1_pub: edge.source ?? null,
+            node2_pub: edge.destination ?? null,
+            capacity: '0', // Will be fetch later
+            node1_policy: {
+              time_lock_delta: edge.cltv_expiry_delta,
+              min_htlc: edge.htlc_minimim_msat,
+              fee_base_msat: edge.fee_base_msat,
+              fee_rate_milli_msat: edge.fee_proportional_millionths,
+              max_htlc_msat: edge.htlc_maximum_msat,
+              last_update: edge.timestamp,
+              disabled: false,          
+            },
+            node2_policy: null,
+          });
+        }
+      }
+    }
+
+    return newGraph;
+  }
+
+  private isIncorrectSnapshot(timestamp, graph): boolean {
+    if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
+        return true;
+    }
+    if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
+      return true;
+    }
+    if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
+      return true;
+    }
+    if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
+      return true;
+    }
+    if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
+      return true;
+    }
+    if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private async $cleanupIncorrectSnapshot(): Promise<void> {
+    // We do not run this one automatically because those stats are not supposed to be inserted in the first
+    // place, but I write them here to remind us we manually run those queries
+
+    // DELETE FROM lightning_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
+    // )
+
+    // DELETE FROM node_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 
+    // )
+  }
+}
+
+export default new LightningStatsImporter;

backend/src/tasks/pools-updater.ts

@@ -8,19 +8,18 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
 
 /**
- * Maintain the most recent version of pools.json
+ * Maintain the most recent version of pools-v2.json
  */
 class PoolsUpdater {
   lastRun: number = 0;
-  currentSha: any = undefined;
-  poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
-  treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
+  currentSha: string | null = null;
+  poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
+  treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
-  constructor() {
-  }
-
-  public async updatePoolsJson() {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+  public async updatePoolsJson(): Promise<void> {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
+      config.MEMPOOL.ENABLED === false
+    ) {
       return;
     }
 
@@ -34,15 +33,9 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`);
-    } else {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`);
-    }
-
     try {
-      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
-      if (githubSha === undefined) {
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
+      if (githubSha === null) {
         return;
       }
 
@@ -50,80 +43,105 @@ class PoolsUpdater {
         this.currentSha = await this.getShaFromDb();
       }
 
-      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
-      if (this.currentSha !== undefined && this.currentSha === githubSha) {
+      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      if (this.currentSha !== null && this.currentSha === githubSha) {
         return;
       }
 
-      if (this.currentSha === undefined) {
-        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`);
+      // See backend README for more details about the mining pools update process
+      if (this.currentSha !== null && // If we don't have any mining pool, download it at least once
+        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
+        !process.env.npm_config_update_pools // We're not manually updating mining pool
+      ) {
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        return;
+      }
+
+      const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
+      if (this.currentSha === null) {
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`);
+        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
         return;
       }
-      await poolsParser.migratePoolsJson(poolsJson);
-      await this.updateDBSha(githubSha);
-      logger.notice('PoolsUpdater completed');
+      poolsParser.setMiningPools(poolsJson);
+
+      if (config.DATABASE.ENABLED === false) { // Don't run db operations
+        logger.info('Mining pools-v2.json import completed (no database)');
+        return;
+      }
+
+      try {
+        await DB.query('START TRANSACTION;');
+        await poolsParser.migratePoolsJson();
+        await this.updateDBSha(githubSha);
+        await DB.query('COMMIT;');
+      } catch (e) {
+        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+        await DB.query('ROLLBACK;');
+      }
+      logger.info('PoolsUpdater completed');
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
-  private async updateDBSha(githubSha: string) {
+  private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
     if (config.DATABASE.ENABLED === true) {
       try {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
+        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       }
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
-  private async getShaFromDb(): Promise<string | undefined> {
+  private async getShaFromDb(): Promise<string | null> {
     try {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
-      return (rows.length > 0 ? rows[0].string : undefined);
+      return (rows.length > 0 ? rows[0].string : null);
     } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
-      return undefined;
+      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      return null;
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from github
+   * Fetch our latest pools-v2.json sha from github
    */
-  private async fetchPoolsSha(): Promise<string | undefined> {
+  private async fetchPoolsSha(): Promise<string | null> {
     const response = await this.query(this.treeUrl);
 
     if (response !== undefined) {
       for (const file of response['tree']) {
-        if (file['path'] === 'pools.json') {
+        if (file['path'] === 'pools-v2.json') {
           return file['sha'];
         }
       }
     }
 
-    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`);
-    return undefined;
+    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
+    return null;
   }
 
   /**
    * Http request wrapper
    */
-  private async query(path): Promise<object | undefined> {
+  private async query(path): Promise<any[] | undefined> {
     type axiosOptions = {
       headers: {
         'User-Agent': string

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -8,12 +8,13 @@ class BitfinexApi implements PriceFeed {
   public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
   public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
 
-  constructor() {
-  }
-
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['last_price'], 10) : -1;
+    if (response && response['last_price']) {
+      return parseInt(response['last_price'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -13,7 +13,11 @@ class BitflyerApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['ltp'], 10) : -1;
+    if (response && response['ltp']) {
+      return parseInt(response['ltp'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -13,7 +13,11 @@ class CoinbaseApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['data']['amount'], 10) : -1;
+    if (response && response['data'] && response['data']['amount']) {
+      return parseInt(response['data']['amount'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/ftx-api.ts

@@ -1,43 +0,0 @@
-import { query } from '../../utils/axios-query';
-import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
-
-class FtxApi implements PriceFeed {
-  public name: string = 'FTX';
-  public currencies: string[] = ['USD', 'BRZ', 'EUR', 'JPY', 'AUD'];
-
-  public url: string = 'https://ftx.com/api/markets/BTC/';
-  public urlHist: string = 'https://ftx.com/api/markets/BTC/{CURRENCY}/candles?resolution={GRANULARITY}';
-
-  constructor() {
-  }
-
-  public async $fetchPrice(currency): Promise<number> {
-    const response = await query(this.url + currency);
-    return response ? parseInt(response['result']['last'], 10) : -1;
-  }
-
-  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
-    const priceHistory: PriceHistory = {};
-
-    for (const currency of currencies) {
-      if (this.currencies.includes(currency) === false) {
-        continue;
-      }
-
-      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency));
-      const pricesRaw = response ? response['result'] : [];
-
-      for (const price of pricesRaw as any[]) {
-        const time = Math.round(price['time'] / 1000);
-        if (priceHistory[time] === undefined) {
-          priceHistory[time] = priceUpdater.getEmptyPricesObj();
-        }
-        priceHistory[time][currency] = price['close'];
-      }
-    }
-
-    return priceHistory;
-  }
-}
-
-export default FtxApi;

backend/src/tasks/price-feeds/gemini-api.ts

@@ -13,7 +13,11 @@ class GeminiApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['last'], 10) : -1;
+    if (response && response['last']) {
+      return parseInt(response['last'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/kraken-api.ts

@@ -23,7 +23,14 @@ class KrakenApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
+    const ticker = this.getTicker(currency);
+    if (response && response['result'] && response['result'][ticker] &&
+      response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0
+    ) {
+      return parseInt(response['result'][ticker]['c'][0], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
@@ -91,7 +98,7 @@ class KrakenApi implements PriceFeed {
     }
 
     if (Object.keys(priceHistory).length > 0) {
-      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
+      logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining);
     }
   }
 }

backend/src/tasks/price-updater.ts

@@ -1,11 +1,11 @@
 import * as fs from 'fs';
+import path from 'path';
 import config from '../config';
 import logger from '../logger';
-import PricesRepository from '../repositories/PricesRepository';
+import PricesRepository, { ApiPrice, MAX_PRICES } from '../repositories/PricesRepository';
 import BitfinexApi from './price-feeds/bitfinex-api';
 import BitflyerApi from './price-feeds/bitflyer-api';
 import CoinbaseApi from './price-feeds/coinbase-api';
-import FtxApi from './price-feeds/ftx-api';
 import GeminiApi from './price-feeds/gemini-api';
 import KrakenApi from './price-feeds/kraken-api';
 
@@ -20,41 +20,36 @@ export interface PriceFeed {
 }
 
 export interface PriceHistory {
-  [timestamp: number]: Prices;
-}
-
-export interface Prices {
-  USD: number;
-  EUR: number;
-  GBP: number;
-  CAD: number;
-  CHF: number;
-  AUD: number;
-  JPY: number;
+  [timestamp: number]: ApiPrice;
 }
 
 class PriceUpdater {
-  historyInserted: boolean = false;
-  lastRun: number = 0;
-  lastHistoricalRun: number = 0;
-  running: boolean = false;
-  feeds: PriceFeed[] = [];
-  currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
-  latestPrices: Prices;
+  public historyInserted = false;
+  private lastRun = 0;
+  private lastHistoricalRun = 0;
+  private running = false;
+  private feeds: PriceFeed[] = [];
+  private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+  private latestPrices: ApiPrice;
+  private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
 
   constructor() {
     this.latestPrices = this.getEmptyPricesObj();
 
     this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
-    this.feeds.push(new FtxApi());
     this.feeds.push(new KrakenApi());
     this.feeds.push(new CoinbaseApi());
     this.feeds.push(new BitfinexApi());
     this.feeds.push(new GeminiApi());
   }
 
-  public getEmptyPricesObj(): Prices {
+  public getLatestPrices(): ApiPrice {
+    return this.latestPrices;
+  }
+
+  public getEmptyPricesObj(): ApiPrice {
     return {
+      time: 0,
       USD: -1,
       EUR: -1,
       GBP: -1,
@@ -65,7 +60,24 @@ class PriceUpdater {
     };
   }
 
+  public setRatesChangedCallback(fn: (rates: ApiPrice) => void): void {
+    this.ratesChangedCallback = fn;
+  }
+
+  /**
+   * We execute this function before the websocket initialization since
+   * the websocket init is not done asyncronously
+   */
+  public async $initializeLatestPriceWithDb(): Promise<void> {
+    this.latestPrices = await PricesRepository.$getLatestConversionRates();
+  }
+
   public async $run(): Promise<void> {
+    if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') {
+      // Coins have no value on testnet/signet, so we want to always show 0
+      return;
+    }
+
     if (this.running === true) {
       return;
     }
@@ -77,13 +89,12 @@ class PriceUpdater {
     }
 
     try {
+      await this.$updatePrice();
       if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
         await this.$insertHistoricalPrices();
-      } else {
-        await this.$updatePrice();
       }
-    } catch (e) {
-      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`);
+    } catch (e: any) {
+      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
     }
 
     this.running = false;
@@ -113,22 +124,26 @@ class PriceUpdater {
         if (feed.currencies.includes(currency)) {
           try {
             const price = await feed.$fetchPrice(currency);
-            if (price > 0) {
+            if (price > -1 && price < MAX_PRICES[currency]) {
               prices.push(price);
             }
-            logger.debug(`${feed.name} BTC/${currency} price: ${price}`);
+            logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
           } catch (e) {
-            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`);
+            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
           }
         }
       }
       if (prices.length === 1) {
-        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`);
+        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`, logger.tags.mining);
       }
 
       // Compute average price, non weighted
       prices = prices.filter(price => price > 0);
-      this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+      if (prices.length === 0) {
+        this.latestPrices[currency] = -1;
+      } else {
+        this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+      }
     }
 
     logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
@@ -145,7 +160,15 @@ class PriceUpdater {
       }
     }
 
+    if (this.ratesChangedCallback) {
+      this.ratesChangedCallback(this.latestPrices);
+    }
+
     this.lastRun = new Date().getTime() / 1000;
+
+    if (this.latestPrices.USD === -1) {
+      this.latestPrices = await PricesRepository.$getLatestConversionRates();
+    }
   }
 
   /**
@@ -158,7 +181,7 @@ class PriceUpdater {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
     // Insert MtGox weekly prices
-    const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
+    const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
     const prices = this.getEmptyPricesObj();
     let insertedCount: number = 0;
     for (const price of pricesJson) {
@@ -176,9 +199,9 @@ class PriceUpdater {
       ++insertedCount;
     }
     if (insertedCount > 0) {
-      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
     } else {
-      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
     }
 
     // Insert Kraken weekly prices
@@ -199,7 +222,7 @@ class PriceUpdater {
   private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
-    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database, this may take a while`);
+    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
 
     const historicalPrices: PriceHistory[] = [];
 
@@ -208,13 +231,13 @@ class PriceUpdater {
       try {
         historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type));
       } catch (e) {
-        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
+        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
       }
     }
 
     // Group them by timestamp and currency, for example
     // grouped[123456789]['USD'] = [1, 2, 3, 4];
-    const grouped: Object = {};
+    const grouped = {};
     for (const historicalEntry of historicalPrices) {
       for (const time in historicalEntry) {
         if (existingPriceTimes.includes(parseInt(time, 10))) {
@@ -229,8 +252,8 @@ class PriceUpdater {
 
         for (const currency of this.currencies) {
           const price = historicalEntry[time][currency];
-          if (price > 0) {
-            grouped[time][currency].push(parseInt(price, 10));
+          if (price > -1 && price < MAX_PRICES[currency]) {
+            grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
           }
         }
       }
@@ -239,7 +262,7 @@ class PriceUpdater {
     // Average prices and insert everything into the db
     let totalInserted = 0;
     for (const time in grouped) {
-      const prices: Prices = this.getEmptyPricesObj();
+      const prices: ApiPrice = this.getEmptyPricesObj();
       for (const currency in grouped[time]) {
         if (grouped[time][currency].length === 0) {
           continue;
@@ -253,9 +276,9 @@ class PriceUpdater {
     }
 
     if (totalInserted > 0) {
-      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
+      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
     } else {
-      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
+      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
     }
   }
 }

backend/src/utils/blocks-utils.ts

@@ -1,33 +0,0 @@
-import { BlockExtended } from '../mempool.interfaces';
-
-export function prepareBlock(block: any): BlockExtended {
-  return <BlockExtended>{
-    id: block.id ?? block.hash, // hash for indexed block
-    timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
-    height: block.height,
-    version: block.version,
-    bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
-    nonce: block.nonce,
-    difficulty: block.difficulty,
-    merkle_root: block.merkle_root ?? block.merkleroot,
-    tx_count: block.tx_count ?? block.nTx,
-    size: block.size,
-    weight: block.weight,
-    previousblockhash: block.previousblockhash,
-    extras: {
-      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
-      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
-      feeRange: block.feeRange ?? block.fee_span,
-      reward: block.reward ?? block?.extras?.reward,
-      totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
-      avgFee: block?.extras?.avgFee ?? block.avg_fee,
-      avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
-      pool: block?.extras?.pool ?? (block?.pool_id ? {
-        id: block.pool_id,
-        name: block.pool_name,
-        slug: block.pool_slug,
-      } : undefined),
-      usd: block?.extras?.usd ?? block.usd ?? null,
-    }
-  };
-}

backend/src/utils/clone.ts

@@ -0,0 +1,14 @@
+// simple recursive deep clone for literal-type objects
+// does not preserve Dates, Maps, Sets etc
+// does not support recursive objects
+// properties deeper than maxDepth will be shallow cloned
+export function deepClone(obj: any, maxDepth: number = 50, depth: number = 0): any {
+  let cloned = obj;
+  if (depth < maxDepth && typeof obj === 'object') {
+    cloned = Array.isArray(obj) ? [] : {};
+    for (const key in obj) {
+      cloned[key] = deepClone(obj[key], maxDepth, depth + 1);
+    }
+  }
+  return cloned;
+}

backend/src/utils/format.ts

@@ -0,0 +1,29 @@
+const byteUnits = ['B', 'kB', 'MB', 'GB', 'TB'];
+
+export function getBytesUnit(bytes: number): string {
+  if (isNaN(bytes) || !isFinite(bytes)) {
+    return 'B';
+  }
+  
+  let unitIndex = 0;
+  while (unitIndex < byteUnits.length && bytes > 1024) {
+    unitIndex++;
+    bytes /= 1024;
+  }
+
+  return byteUnits[unitIndex];
+}
+
+export function formatBytes(bytes: number, toUnit: string, skipUnit = false): string {
+  if (isNaN(bytes) || !isFinite(bytes)) {
+    return `${bytes}`;
+  }
+  
+  let unitIndex = 0;
+  while (unitIndex < byteUnits.length && (toUnit && byteUnits[unitIndex] !== toUnit || (!toUnit && bytes > 1024))) {
+    unitIndex++;
+    bytes /= 1024;
+  }
+
+  return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`;
+}

backend/src/utils/ipcheck.js

@@ -0,0 +1,119 @@
+var net = require('net');
+
+var IPCheck = module.exports = function(input) {
+  var self = this;
+
+  if (!(self instanceof IPCheck)) {
+    return new IPCheck(input);
+  }
+
+  self.input = input;
+  self.parse();
+};
+
+IPCheck.prototype.parse = function() {
+  var self = this;
+
+  if (!self.input || typeof self.input !== 'string') return self.valid = false;
+
+  var ip;
+
+  var pos = self.input.lastIndexOf('/');
+  if (pos !== -1) {
+    ip = self.input.substring(0, pos);
+    self.mask = +self.input.substring(pos + 1);
+  } else {
+    ip = self.input;
+    self.mask = null;
+  }
+
+  self.ipv = net.isIP(ip);
+  self.valid = !!self.ipv && !isNaN(self.mask);
+
+  if (!self.valid) return;
+
+  // default mask = 32 for ipv4 and 128 for ipv6
+  if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
+
+  if (self.ipv === 4) {
+    // difference between ipv4 and ipv6 masks
+    self.mask += 96;
+  }
+
+  if (self.mask < 0 || self.mask > 128) {
+    self.valid = false;
+    return;
+  }
+
+  self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
+
+  if(self.ipv === 4){
+    self.parseIPv4(ip);
+  }else{
+    self.parseIPv6(ip);
+  }
+};
+
+IPCheck.prototype.parseIPv4 = function(ip) {
+  var self = this;
+
+  // ipv4 addresses live under ::ffff:0:0
+  self.address[10] = self.address[11] = 0xff;
+
+  var octets = ip.split('.');
+  for (var i = 0; i < 4; i++) {
+    self.address[i + 12] = parseInt(octets[i], 10);
+  }
+};
+
+
+var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
+
+IPCheck.prototype.parseIPv6 = function(ip) {
+  var self = this;
+
+  var transitionalMatch = V6_TRANSITIONAL.exec(ip);
+  if(transitionalMatch){
+    self.parseIPv4(transitionalMatch[1]);
+    return;
+  }
+
+  var bits = ip.split(':');
+  if (bits.length < 8) {
+    ip = ip.replace('::', Array(11 - bits.length).join(':'));
+    bits = ip.split(':');
+  }
+
+  var j = 0;
+  for (var i = 0; i < bits.length; i += 1) {
+    var x = bits[i] ? parseInt(bits[i], 16) : 0;
+    self.address[j++] = x >> 8;
+    self.address[j++] = x & 0xff;
+  }
+};
+
+IPCheck.prototype.match = function(cidr) {
+  var self = this;
+
+  if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
+  if (!self.valid || !cidr.valid) return false;
+
+  var mask = cidr.mask;
+  var i = 0;
+
+  while (mask >= 8) {
+    if (self.address[i] !== cidr.address[i]) return false;
+
+    i++;
+    mask -= 8;
+  }
+
+  var shift = 8 - mask;
+  return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
+};
+
+
+IPCheck.match = function(ip, cidr) {
+  ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
+  return ip.match(cidr);
+};

backend/src/utils/pairing-heap.ts

@@ -0,0 +1,174 @@
+export type HeapNode<T> = {
+  element: T
+  child?: HeapNode<T>
+  next?: HeapNode<T>
+  prev?: HeapNode<T>
+} | null | undefined;
+
+// minimal pairing heap priority queue implementation
+export class PairingHeap<T> {
+  private root: HeapNode<T> = null;
+  private comparator: (a: T, b: T) => boolean;
+
+  // comparator function should return 'true' if a is higher priority than b
+  constructor(comparator: (a: T, b: T) => boolean) {
+    this.comparator = comparator;
+  }
+
+  isEmpty(): boolean {
+    return !this.root;
+  }
+
+  add(element: T): HeapNode<T> {
+    const node: HeapNode<T> = {
+      element
+    };
+
+    this.root = this.meld(this.root, node);
+
+    return node;
+  }
+
+  // returns the top priority element without modifying the queue
+  peek(): T | void {
+    return this.root?.element;
+  }
+
+  // removes and returns the top priority element
+  pop(): T | void {
+    let element;
+    if (this.root) {
+      const node = this.root;
+      element = node.element;
+      this.root = this.mergePairs(node.child);
+    }
+    return element;
+  }
+
+  deleteNode(node: HeapNode<T>): void {
+    if (!node) {
+      return;
+    }
+
+    if (node === this.root) {
+      this.root = this.mergePairs(node.child);
+    }
+    else {
+      if (node.prev) {
+        if (node.prev.child === node) {
+          node.prev.child = node.next;
+        }
+        else {
+          node.prev.next = node.next;
+        }
+      }
+      if (node.next) {
+        node.next.prev = node.prev;
+      }
+      this.root = this.meld(this.root, this.mergePairs(node.child));
+    }
+
+    node.child = null;
+    node.prev = null;
+    node.next = null;
+  }
+
+  // fix the heap after increasing the priority of a given node
+  increasePriority(node: HeapNode<T>): void {
+    // already the top priority element
+    if (!node || node === this.root) {
+      return;
+    }
+    // extract from siblings
+    if (node.prev) {
+      if (node.prev?.child === node) {
+        if (this.comparator(node.prev.element, node.element)) {
+          // already in a valid position
+          return;
+        }
+        node.prev.child = node.next;
+      }
+      else {
+        node.prev.next = node.next;
+      }
+    }
+    if (node.next) {
+      node.next.prev = node.prev;
+    }
+
+    this.root = this.meld(this.root, node);
+  }
+
+  decreasePriority(node: HeapNode<T>): void {
+    this.deleteNode(node);
+    this.root = this.meld(this.root, node);
+  }
+
+  meld(a: HeapNode<T>, b: HeapNode<T>): HeapNode<T> {
+    if (!a) {
+      return b;
+    }
+    if (!b || a === b) {
+      return a;
+    }
+
+    let parent: HeapNode<T> = b;
+    let child: HeapNode<T> = a;
+    if (this.comparator(a.element, b.element)) {
+      parent = a;
+      child = b;
+    }
+
+    child.next = parent.child;
+    if (parent.child) {
+      parent.child.prev = child;
+    }
+    child.prev = parent;
+    parent.child = child;
+
+    parent.next = null;
+    parent.prev = null;
+
+    return parent;
+  }
+
+  mergePairs(node: HeapNode<T>): HeapNode<T> {
+    if (!node) {
+      return null;
+    }
+
+    let current: HeapNode<T> = node;
+    let next: HeapNode<T>;
+    let nextCurrent: HeapNode<T>;
+    let pairs: HeapNode<T>;
+    let melded: HeapNode<T>;
+    while (current) {
+      next = current.next;
+      if (next) {
+        nextCurrent = next.next;
+        melded = this.meld(current, next);
+        if (melded) {
+          melded.prev = pairs;
+        }
+        pairs = melded;
+      }
+      else {
+        nextCurrent = null;
+        current.prev = pairs;
+        pairs = current;
+        break;
+      }
+      current = nextCurrent;
+    }
+
+    melded = null;
+    let prev: HeapNode<T>;
+    while (pairs) {
+      prev = pairs.prev;
+      melded = this.meld(melded, pairs);
+      pairs = prev;
+    }
+
+    return melded;
+  }
+}

backend/testSetup.ts

@@ -0,0 +1,5 @@
+jest.mock('./mempool-config.json', () => ({}), { virtual: true });
+jest.mock('./src/logger.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true });

backend/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig",
+  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"],
+  "compilerOptions": {
+    "types": ["node"]
+  },
+
+}