Use p-limit to limit concurrent requests

Fix: calcBitsDifference regtest fix

.github/dependabot.yml

@@ -1,6 +1,7 @@
 version: 2
 updates:
   - package-ecosystem: npm
+    versioning-strategy: increase
     directory: "/backend"
     schedule:
       interval: daily
@@ -14,6 +15,21 @@ updates:
 
   - package-ecosystem: npm
     directory: "/frontend"
+    versioning-strategy: increase
+    groups:
+      frontend-angular-dependencies:
+        patterns:
+          - "@angular*"
+          - "@ng-*"
+          - "ngx-*"
+      frontend-jest-dependencies:
+        patterns:
+          - "@types/jest"
+          - "jest"
+      frontend-eslint-dependencies:
+        patterns:
+          - "@typescript-eslint*"
+          - "eslint"
     schedule:
       interval: daily
     open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
 
       - name: Unit Tests
         if: ${{ matrix.flavor == 'dev'}}
-        run: npm run test
+        run: npm run test:ci
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Build

.github/workflows/cypress.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v3
         with:
-          node-version: 16.15.0
+          node-version: 18
           cache: "npm"
           cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 

LICENSE

@@ -13,7 +13,7 @@ the terms of (at your option) either:
  proxy statement published on <https://mempool.space/about>.
 
 However, this copyright license does not include an implied right or license to
-use our trademarks: The Mempool Open Source Project™, mempool.space™, the
+use our trademarks: The Mempool Open Source Project®, mempool.space™, the
 mempool Logo™, the mempool.space Vertical Logo™, the mempool.space Horizontal
 Logo™, the mempool Square Logo™, and the mempool Blocks logo™ are registered
 trademarks or trademarks of Mempool Space K.K in Japan, the United States,

README.md

@@ -1,4 +1,4 @@
-# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
+# The Mempool Open Source Project® [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
 https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
 

backend/.gitignore

@@ -45,3 +45,6 @@ testem.log
 #System Files
 .DS_Store
 Thumbs.db
+
+# package folder (npm run package output)
+/package

backend/mempool-config.sample.json

@@ -8,6 +8,7 @@
     "API_URL_PREFIX": "/api/v1/",
     "POLL_RATE_MS": 2000,
     "CACHE_DIR": "./cache",
+    "CACHE_ENABLED": true,
     "CLEAR_PROTECTION_MINUTES": 20,
     "RECOMMENDED_FEE_PERCENTILE": 50,
     "BLOCK_WEIGHT_UNITS": 4000000,
@@ -31,7 +32,8 @@
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATES_PER_HOUR": 1
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -48,7 +50,8 @@
   "ESPLORA": {
     "REST_API_URL": "http://127.0.0.1:3000",
     "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
-    "RETRY_UNIX_SOCKET_AFTER": 30000
+    "RETRY_UNIX_SOCKET_AFTER": 30000,
+    "FALLBACK": []
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -114,10 +117,6 @@
     "USERNAME": "",
     "PASSWORD": ""
   },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
-    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "https://mempool.space/api/v1",
     "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
@@ -136,5 +135,9 @@
       "trusted",
       "servers"
     ]
+  },
+  "MEMPOOL_SERVICES": {
+    "API": "https://mempool.space/api",
+    "ACCELERATIONS": false
   }
 }

backend/npm_package.sh

@@ -0,0 +1,17 @@
+#/bin/sh
+set -e
+
+# Remove previous dist folder
+rm -rf dist
+# Build new dist folder
+npm run build
+# Remove previous package folder
+rm -rf package
+# Move JS and deps
+mv dist package
+cp -R node_modules package
+# Remove symlink for rust-gbt and insert real folder
+rm package/node_modules/rust-gbt
+cp -R rust-gbt package/node_modules
+# Clean up deps
+npm run package-rm-build-deps

backend/npm_package_rm_build_deps.sh

@@ -0,0 +1,12 @@
+#/bin/sh
+set -e
+
+# Cleaning up inside the node_modules folder
+cd package/node_modules
+rm -r \
+  typescript \
+  @typescript-eslint \
+  @napi-rs \
+  ./rust-gbt/src \
+  ./rust-gbt/Cargo.toml \
+  ./rust-gbt/build.rs

backend/package-lock.json

@@ -17,7 +17,8 @@
         "crypto-js": "~4.1.1",
         "express": "~4.18.2",
         "maxmind": "~4.3.11",
-        "mysql2": "~3.5.2",
+        "mysql2": "~3.6.0",
+        "redis": "^4.6.6",
         "rust-gbt": "file:./rust-gbt",
         "socks-proxy-agent": "~7.0.0",
         "typescript": "~4.9.3",
@@ -1555,6 +1556,64 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@redis/bloom": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
+      "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/client": {
+      "version": "1.5.7",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz",
+      "integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==",
+      "dependencies": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@redis/client/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+    },
+    "node_modules/@redis/graph": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
+      "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/json": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
+      "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/search": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz",
+      "integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/time-series": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
+      "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
     "node_modules/@sinclair/typebox": {
       "version": "0.25.24",
       "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz",
@@ -2718,6 +2777,14 @@
         "node": ">=12"
       }
     },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -3678,6 +3745,14 @@
         "is-property": "^1.0.2"
       }
     },
+    "node_modules/generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/gensync": {
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -6027,9 +6102,9 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node_modules/mysql2": {
-      "version": "3.5.2",
-      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz",
-      "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz",
+      "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==",
       "dependencies": {
         "denque": "^2.1.0",
         "generate-function": "^2.3.1",
@@ -6577,6 +6652,19 @@
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
       "dev": true
     },
+    "node_modules/redis": {
+      "version": "4.6.6",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz",
+      "integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==",
+      "dependencies": {
+        "@redis/bloom": "1.2.0",
+        "@redis/client": "1.5.7",
+        "@redis/graph": "1.1.0",
+        "@redis/json": "1.0.4",
+        "@redis/search": "1.1.2",
+        "@redis/time-series": "1.0.4"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -8704,6 +8792,53 @@
         "fastq": "^1.6.0"
       }
     },
+    "@redis/bloom": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
+      "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
+      "requires": {}
+    },
+    "@redis/client": {
+      "version": "1.5.7",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz",
+      "integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==",
+      "requires": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "dependencies": {
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+        }
+      }
+    },
+    "@redis/graph": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
+      "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
+      "requires": {}
+    },
+    "@redis/json": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
+      "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
+      "requires": {}
+    },
+    "@redis/search": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz",
+      "integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==",
+      "requires": {}
+    },
+    "@redis/time-series": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
+      "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
+      "requires": {}
+    },
     "@sinclair/typebox": {
       "version": "0.25.24",
       "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz",
@@ -9604,6 +9739,11 @@
         "wrap-ansi": "^7.0.0"
       }
     },
+    "cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="
+    },
     "co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -10332,6 +10472,11 @@
         "is-property": "^1.0.2"
       }
     },
+    "generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g=="
+    },
     "gensync": {
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -12067,9 +12212,9 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "mysql2": {
-      "version": "3.5.2",
-      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz",
-      "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz",
+      "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==",
       "requires": {
         "denque": "^2.1.0",
         "generate-function": "^2.3.1",
@@ -12454,6 +12599,19 @@
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
       "dev": true
     },
+    "redis": {
+      "version": "4.6.6",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz",
+      "integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==",
+      "requires": {
+        "@redis/bloom": "1.2.0",
+        "@redis/client": "1.5.7",
+        "@redis/graph": "1.1.0",
+        "@redis/json": "1.0.4",
+        "@redis/search": "1.1.2",
+        "@redis/time-series": "1.0.4"
+      }
+    },
     "require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",

backend/package.json

@@ -22,19 +22,20 @@
   "main": "index.ts",
   "scripts": {
     "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
-    "build": "npm run build-rust && npm run tsc && npm run create-resources",
+    "build": "npm run rust-build && npm run tsc && npm run create-resources",
     "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
-    "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && mv rust-gbt package && npm run package-rm-build-deps",
-    "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint @napi-rs ../rust-gbt/target ../rust-gbt/node_modules ../rust-gbt/src)",
+    "package": "./npm_package.sh",
+    "package-rm-build-deps": "./npm_package_rm_build_deps.sh",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=16384 dist/index.js",
     "reindex-updated-pools": "npm run start-production --update-pools",
     "reindex-all-blocks": "npm run start-production --update-pools --reindex-blocks",
     "test": "./node_modules/.bin/jest --coverage",
+    "test:ci": "CI=true ./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"",
-    "build-rust": "cd rust-gbt && npm install"
+    "rust-build": "cd rust-gbt && npm run build-release"
   },
   "dependencies": {
     "@babel/core": "^7.21.3",
@@ -45,15 +46,16 @@
     "crypto-js": "~4.1.1",
     "express": "~4.18.2",
     "maxmind": "~4.3.11",
-    "mysql2": "~3.5.2",
+    "mysql2": "~3.6.0",
     "rust-gbt": "file:./rust-gbt",
+    "redis": "^4.6.6",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.9.3",
     "ws": "~8.13.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.21.3",
     "@babel/code-frame": "^7.18.6",
+    "@babel/core": "^7.21.3",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.17",

backend/rust-gbt/index.d.ts

@@ -12,6 +12,10 @@ export interface ThreadTransaction {
   effectiveFeePerVsize: number
   inputs: Array<number>
 }
+export interface ThreadAcceleration {
+  uid: number
+  delta: number
+}
 export class GbtGenerator {
   constructor()
   /**
@@ -19,13 +23,13 @@ export class GbtGenerator {
    *
    * Rejects if the thread panics or if the Mutex is poisoned.
    */
-  make(mempool: Array<ThreadTransaction>, maxUid: number): Promise<GbtResult>
+  make(mempool: Array<ThreadTransaction>, accelerations: Array<ThreadAcceleration>, maxUid: number): Promise<GbtResult>
   /**
    * # Errors
    *
    * Rejects if the thread panics or if the Mutex is poisoned.
    */
-  update(newTxs: Array<ThreadTransaction>, removeTxs: Array<number>, maxUid: number): Promise<GbtResult>
+  update(newTxs: Array<ThreadTransaction>, removeTxs: Array<number>, accelerations: Array<ThreadAcceleration>, maxUid: number): Promise<GbtResult>
 }
 /**
  * The result from calling the gbt function.

backend/rust-gbt/src/audit_transaction.rs

@@ -1,6 +1,6 @@
 use crate::{
     u32_hasher_types::{u32hashset_new, U32HasherState},
-    ThreadTransaction,
+    ThreadTransaction, thread_acceleration::ThreadAcceleration,
 };
 use std::{
     cmp::Ordering,
@@ -88,44 +88,49 @@ impl Ord for AuditTransaction {
 }
 
 #[inline]
-fn calc_fee_rate(fee: f64, vsize: f64) -> f64 {
-    fee / (if vsize == 0.0 { 1.0 } else { vsize })
+fn calc_fee_rate(fee: u64, vsize: f64) -> f64 {
+    (fee as f64) / (if vsize == 0.0 { 1.0 } else { vsize })
 }
 
 impl AuditTransaction {
-    pub fn from_thread_transaction(tx: &ThreadTransaction) -> Self {
+    pub fn from_thread_transaction(tx: &ThreadTransaction, maybe_acceleration: Option<Option<&ThreadAcceleration>>) -> Self {
+        let fee_delta = match maybe_acceleration {
+            Some(Some(acceleration)) => acceleration.delta,
+            _ => 0.0
+        };
+        let fee = (tx.fee as u64) + (fee_delta as u64);
         // rounded up to the nearest integer
         let is_adjusted = tx.weight < (tx.sigops * 20);
         let sigop_adjusted_vsize = ((tx.weight + 3) / 4).max(tx.sigops * 5);
         let sigop_adjusted_weight = tx.weight.max(tx.sigops * 20);
-        let effective_fee_per_vsize = if is_adjusted {
-            calc_fee_rate(tx.fee, f64::from(sigop_adjusted_weight) / 4.0)
+        let effective_fee_per_vsize = if is_adjusted || fee_delta > 0.0 {
+            calc_fee_rate(fee, f64::from(sigop_adjusted_weight) / 4.0)
         } else {
             tx.effective_fee_per_vsize
         };
         Self {
             uid: tx.uid,
             order: tx.order,
-            fee: tx.fee as u64,
+            fee,
             weight: tx.weight,
             sigop_adjusted_weight,
             sigop_adjusted_vsize,
             sigops: tx.sigops,
-            adjusted_fee_per_vsize: calc_fee_rate(tx.fee, f64::from(sigop_adjusted_vsize)),
+            adjusted_fee_per_vsize: calc_fee_rate(fee, f64::from(sigop_adjusted_vsize)),
             effective_fee_per_vsize,
             dependency_rate: f64::INFINITY,
             inputs: tx.inputs.clone(),
             relatives_set_flag: false,
             ancestors: u32hashset_new(),
             children: u32hashset_new(),
-            ancestor_fee: tx.fee as u64,
+            ancestor_fee: fee,
             ancestor_sigop_adjusted_weight: sigop_adjusted_weight,
             ancestor_sigop_adjusted_vsize: sigop_adjusted_vsize,
             ancestor_sigops: tx.sigops,
             score: 0.0,
             used: false,
             modified: false,
-            dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize,
+            dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize || fee_delta > 0.0,
         }
     }
 
@@ -156,7 +161,7 @@ impl AuditTransaction {
         // grows, so if we think of 0 as "grew infinitely" then dependency_rate would be
         // the smaller of the two. If either side is NaN, the other side is returned.
         self.dependency_rate.min(calc_fee_rate(
-            self.ancestor_fee as f64,
+            self.ancestor_fee,
             f64::from(self.ancestor_sigop_adjusted_weight) / 4.0,
         ))
     }
@@ -172,7 +177,7 @@ impl AuditTransaction {
     #[inline]
     fn calc_new_score(&mut self) {
         self.score = self.adjusted_fee_per_vsize.min(calc_fee_rate(
-            self.ancestor_fee as f64,
+            self.ancestor_fee,
             f64::from(self.ancestor_sigop_adjusted_vsize),
         ));
     }

backend/rust-gbt/src/gbt.rs

@@ -5,7 +5,7 @@ use tracing::{info, trace};
 use crate::{
     audit_transaction::{partial_cmp_uid_score, AuditTransaction},
     u32_hasher_types::{u32hashset_new, u32priority_queue_with_capacity, U32HasherState},
-    GbtResult, ThreadTransactionsMap,
+    GbtResult, ThreadTransactionsMap, thread_acceleration::ThreadAcceleration,
 };
 
 const MAX_BLOCK_WEIGHT_UNITS: u32 = 4_000_000 - 4_000;
@@ -53,7 +53,13 @@ impl Ord for TxPriority {
 // TODO: Make gbt smaller to fix these lints.
 #[allow(clippy::too_many_lines)]
 #[allow(clippy::cognitive_complexity)]
-pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult {
+pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAcceleration], max_uid: usize) -> GbtResult {
+    let mut indexed_accelerations = Vec::with_capacity(max_uid + 1);
+    indexed_accelerations.resize(max_uid + 1, None);
+    for acceleration in accelerations {
+        indexed_accelerations[acceleration.uid as usize] = Some(acceleration);
+    }
+
     let mempool_len = mempool.len();
     let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1);
     audit_pool.resize(max_uid + 1, None);
@@ -63,7 +69,8 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult {
 
     info!("Initializing working structs");
     for (uid, tx) in &mut *mempool {
-        let audit_tx = AuditTransaction::from_thread_transaction(tx);
+        let acceleration = indexed_accelerations.get(*uid as usize);
+        let audit_tx = AuditTransaction::from_thread_transaction(tx, acceleration.copied());
         // Safety: audit_pool and mempool_stack must always contain the same transactions
         audit_pool[*uid as usize] = Some(ManuallyDrop::new(audit_tx));
         mempool_stack.push(*uid);

backend/rust-gbt/src/lib.rs

@@ -9,6 +9,7 @@
 use napi::bindgen_prelude::Result;
 use napi_derive::napi;
 use thread_transaction::ThreadTransaction;
+use thread_acceleration::ThreadAcceleration;
 use tracing::{debug, info, trace};
 use tracing_log::LogTracer;
 use tracing_subscriber::{EnvFilter, FmtSubscriber};
@@ -19,6 +20,7 @@ use std::sync::{Arc, Mutex};
 mod audit_transaction;
 mod gbt;
 mod thread_transaction;
+mod thread_acceleration;
 mod u32_hasher_types;
 
 use u32_hasher_types::{u32hashmap_with_capacity, U32HasherState};
@@ -74,10 +76,11 @@ impl GbtGenerator {
     ///
     /// Rejects if the thread panics or if the Mutex is poisoned.
     #[napi]
-    pub async fn make(&self, mempool: Vec<ThreadTransaction>, max_uid: u32) -> Result<GbtResult> {
+    pub async fn make(&self, mempool: Vec<ThreadTransaction>, accelerations: Vec<ThreadAcceleration>, max_uid: u32) -> Result<GbtResult> {
         trace!("make: Current State {:#?}", self.thread_transactions);
         run_task(
             Arc::clone(&self.thread_transactions),
+            accelerations,
             max_uid as usize,
             move |map| {
                 for tx in mempool {
@@ -96,11 +99,13 @@ impl GbtGenerator {
         &self,
         new_txs: Vec<ThreadTransaction>,
         remove_txs: Vec<u32>,
+        accelerations: Vec<ThreadAcceleration>,
         max_uid: u32,
     ) -> Result<GbtResult> {
         trace!("update: Current State {:#?}", self.thread_transactions);
         run_task(
             Arc::clone(&self.thread_transactions),
+            accelerations,
             max_uid as usize,
             move |map| {
                 for tx in new_txs {
@@ -141,6 +146,7 @@ pub struct GbtResult {
 /// to the `HashMap` as the only argument. (A move closure is recommended to meet the bounds)
 async fn run_task<F>(
     thread_transactions: Arc<Mutex<ThreadTransactionsMap>>,
+    accelerations: Vec<ThreadAcceleration>,
     max_uid: usize,
     callback: F,
 ) -> Result<GbtResult>
@@ -159,7 +165,7 @@ where
         callback(&mut map);
 
         info!("Starting gbt algorithm for {} elements...", map.len());
-        let result = gbt::gbt(&mut map, max_uid);
+        let result = gbt::gbt(&mut map, &accelerations, max_uid);
         info!("Finished gbt algorithm for {} elements...", map.len());
 
         debug!(

backend/rust-gbt/src/thread_acceleration.rs

@@ -0,0 +1,8 @@
+use napi_derive::napi;
+
+#[derive(Debug)]
+#[napi(object)]
+pub struct ThreadAcceleration {
+    pub uid: u32,
+    pub delta: f64, // fee delta
+}

backend/src/__fixtures__/mempool-config.template.json

@@ -10,6 +10,7 @@
     "AUTOMATIC_BLOCK_REINDEXING": false,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CACHE_ENABLED": true,
     "CLEAR_PROTECTION_MINUTES": 4,
     "RECOMMENDED_FEE_PERCENTILE": 5,
     "BLOCK_WEIGHT_UNITS": 6,
@@ -22,8 +23,8 @@
     "USER_AGENT": "__MEMPOOL_USER_AGENT__",
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": 14,
-    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
     "AUDIT": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": true,
@@ -32,7 +33,8 @@
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATES_PER_HOUR": 1
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -49,7 +51,8 @@
   "ESPLORA": {
     "REST_API_URL": "__ESPLORA_REST_API_URL__",
     "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
-    "RETRY_UNIX_SOCKET_AFTER": 888
+    "RETRY_UNIX_SOCKET_AFTER": 888,
+    "FALLBACK": []
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
@@ -91,10 +94,6 @@
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
     "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
   },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
-    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
     "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
@@ -127,5 +126,13 @@
     "AUDIT": false,
     "AUDIT_START_HEIGHT": 774000,
     "SERVERS": []
+  },
+  "MEMPOOL_SERVICES": {
+    "API": "",
+    "ACCELERATIONS": false
+  },
+  "REDIS": {
+    "ENABLED": false,
+    "UNIX_SOCKET_PATH": "/tmp/redis.sock"
   }
 }

backend/src/__tests__/api/common.ts

@@ -0,0 +1,24 @@
+import { Common } from '../../api/common';
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+
+const randomTransactions = require('./test-data/transactions-random.json');
+const replacedTransactions = require('./test-data/transactions-replaced.json');
+const rbfTransactions = require('./test-data/transactions-rbfs.json');
+
+describe('Mempool Utils', () => {
+  test('should detect RBF transactions with fast method', () => {
+    const newTransactions = rbfTransactions.concat(randomTransactions);
+    const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions);
+    expect(Object.values(result).length).toEqual(2);
+    expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+    expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+  });
+
+  test.only('should detect RBF transactions with scalable method', () => {
+    const newTransactions = rbfTransactions.concat(randomTransactions);
+    const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions, true);
+    expect(Object.values(result).length).toEqual(2);
+    expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+    expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+  });
+});

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -1,4 +1,8 @@
-import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+import {
+  calcBitsDifference,
+  calcDifficultyAdjustment,
+  DifficultyAdjustment,
+} from '../../api/difficulty-adjustment';
 
 describe('Mempool Difficulty Adjustment', () => {
   test('should calculate Difficulty Adjustments properly', () => {
@@ -86,4 +90,46 @@ describe('Mempool Difficulty Adjustment', () => {
       expect(result).toStrictEqual(vector[1]);
     }
   });
+
+  test('should calculate Difficulty change from bits fields of two blocks', () => {
+    // Check same exponent + check min max for output
+    expect(calcBitsDifference(0x1d000200, 0x1d000100)).toEqual(100);
+    expect(calcBitsDifference(0x1d000400, 0x1d000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000800, 0x1d000100)).toEqual(300); // Actually 700
+    expect(calcBitsDifference(0x1d000100, 0x1d000200)).toEqual(-50);
+    expect(calcBitsDifference(0x1d000100, 0x1d000400)).toEqual(-75);
+    expect(calcBitsDifference(0x1d000100, 0x1d000800)).toEqual(-75); // Actually -87.5
+    // Check new higher exponent
+    expect(calcBitsDifference(0x1c000200, 0x1d000001)).toEqual(100);
+    expect(calcBitsDifference(0x1c000400, 0x1d000001)).toEqual(300);
+    expect(calcBitsDifference(0x1c000800, 0x1d000001)).toEqual(300);
+    expect(calcBitsDifference(0x1c000100, 0x1d000002)).toEqual(-50);
+    expect(calcBitsDifference(0x1c000100, 0x1d000004)).toEqual(-75);
+    expect(calcBitsDifference(0x1c000100, 0x1d000008)).toEqual(-75);
+    // Check new lower exponent
+    expect(calcBitsDifference(0x1d000002, 0x1c000100)).toEqual(100);
+    expect(calcBitsDifference(0x1d000004, 0x1c000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000008, 0x1c000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000001, 0x1c000200)).toEqual(-50);
+    expect(calcBitsDifference(0x1d000001, 0x1c000400)).toEqual(-75);
+    expect(calcBitsDifference(0x1d000001, 0x1c000800)).toEqual(-75);
+    // Check error when exponents are too far apart
+    expect(() => calcBitsDifference(0x1d000001, 0x1a000800)).toThrow(
+      /Impossible exponent difference/
+    );
+    // Check invalid inputs
+    expect(() => calcBitsDifference(0x7f000001, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0, 0x1a000800)).toThrow(/Invalid bits/);
+    expect(() => calcBitsDifference(100.2783, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0x00800000, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0x1c000000, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+  });
 });

backend/src/__tests__/api/test-data/transactions-random.json

@@ -0,0 +1,277 @@
+[
+    {
+        "txid": "13f007241d78e8b0b4e57d2ae3fd37bcfe3226534d7cadeba5a549860d960db0",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "cb8f206f4e88bec97107089f3e9e61d50cde53d4541992ae19759b71103cf75c",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014fd6d15ff832c12f1ff04a5ccd5039f7227b260bd",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 fd6d15ff832c12f1ff04a5ccd5039f7227b260bd",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1ql4k3tlur9sf0rlcy5hxd2qulwgnmyc9akehvth",
+                    "value": 610677
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304302205c430b36ebd2bb327951d83440af1f58f127871b2baada4c4dde2bc0b6721f56021f3445099f1a40e35baeda32e8e3727b505ffba0d882b11f498c7762f4184e9901",
+                    "0236b5edd4fbbcfb045960e42ec8a9968944084785932e32940e8cd2583b37da67"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483648
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd",
+                "value": 344697
+            },
+            {
+                "scriptpubkey": "00147dee8a7a38abbfb00dbfba365c8d6712934cc491",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7dee8a7a38abbfb00dbfba365c8d6712934cc491",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1q0hhg573c4wlmqrdlhgm9ert8z2f5e3y3lf9hvx",
+                "value": 265396
+            }
+        ],
+        "size": 224,
+        "weight": 572,
+        "fee": 584,
+        "status": {
+            "confirmed": false
+        },
+        "order": 2953680397,
+        "vsize": 143,
+        "adjustedVsize": 143,
+        "sigops": 5,
+        "feePerVsize": 4.083916083916084,
+        "adjustedFeePerVsize": 4.083916083916084,
+        "effectiveFeePerVsize": 4.083916083916084,
+        "firstSeen": 1691222538,
+        "uid": 526973,
+        "inputs": [
+            526728
+        ],
+        "position": {
+            "block": 7,
+            "vsize": 21429708.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "8e89b20f8a7fadb0e4cdbe57a00eee224f5076bac5387fc276916724e7c4a16a",
+        "version": 2,
+        "locktime": 800571,
+        "vin": [
+            {
+                "txid": "35e16762459539f3a8e52c5dee6a9ccaa9e9268efed33aa2c6e1b7805e849f24",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014d4f16ef275b3e1c4a4ecbef55a164933e0f6460f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d4f16ef275b3e1c4a4ecbef55a164933e0f6460f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q6nckaun4k0suff8vhm6459jfx0s0v3s0ff4ukl",
+                    "value": 1528924
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022019008b26e885bb43da25a11ffac147a057722072eedb68411f114f6e7eb82ebc02201b618264bb97756b88fc3bbc365b73044ac18b33b1067e31cfd5bcd0f50ed2c701",
+                    "039b71145070bd3e8af28e27fa577f2e12ab6bb4e212d3eeaef08b4bc39e8cbc13"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "67c27ed0f767526234bcd5f795a31fab8ec4d0251bf12c68f2746951f4110d90",
+                "vout": 3,
+                "prevout": {
+                    "scriptpubkey": "0014a7c3d613b321375054b2ac9b6114367bc034ad6f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a7c3d613b321375054b2ac9b6114367bc034ad6f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5lpavyanyym4q49j4jdkz9pk00qrftt0yqzvk3",
+                    "value": 436523
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204e67285fc656bc45ed082499b076d5dba2fa21d0d7e64a0ae52b19d69a11760002200f037d81ee540b74397844513b72b08ed92b06db76bd20b08f7a0a3b36ab13d501",
+                    "02a3ebae85f0225b6fbb5ff060afce683a4683507a57544605a29ee7d287e591b4"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "21c38fb9a2521e438c614f53b19ddd7a5594bcc4b77480e762fd4b702fad3374",
+                "vout": 1,
+                "prevout": {
+                    "scriptpubkey": "00149660e34ef88106536c816c037b5b28dd64a812e2",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 9660e34ef88106536c816c037b5b28dd64a812e2",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qjeswxnhcsyr9xmypdsphkkegm4j2syhztgzxv4",
+                    "value": 758149
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022021b556f0aa99329076bcc435338aceaf534963efcab306931b1b2b0461e16e0c02203a78942a3745c4da656bddfd8cf16b85dc04d652904e88682127cdd9ca63339001",
+                    "0298963be4a8f66aca9fcf1c6dc95547aeaa82347543190c91e094c2321142b9f0"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "aa998dbae65240a7386bf7d468459551d99c3de8e2f9057ff5f2d38e17daf788",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "00147bb7413a39943b21ded98ad5e6ad7a222d273e17",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7bb7413a39943b21ded98ad5e6ad7a222d273e17",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q0wm5zw3ejsajrhke3t27dtt6ygkjw0sh9lltg6",
+                    "value": 1067200
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402205e2269f7d4ee0513b34354c38e920aef2dabac6f4350afb2dd105ff3ee43ae7b02202870322f2cb85cb0b2b0e38152f018bfff271dc3ec5aed0515854d0b259aaf3d01",
+                    "03b87320cf3263a644a0d3f89c1b4a7304d9dfda9eb8c891560716abcb73e88b99"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "230253d195d779d4688ba16993985cd27b2e7a687d8b889b3bc63f19ece36f20",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001439647bd997819d12dfc72b0fb9ff9ffcb84946f8",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 39647bd997819d12dfc72b0fb9ff9ffcb84946f8",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q89j8hkvhsxw39h789v8mnlulljuyj3hc9zve97",
+                    "value": 361950
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204f7ca868bb9b92a07fecdc6b9dd56e4e1d007ca1405952f98ed6bc416345b5f2022055320a97791417abf6628fcf6513ac5785b06c630f854d8595e96ea06c3841d301",
+                    "03a3ffe8e3ef2eea129b227e9658164bae0a6d21c17da6de9973ba34d9e04b21a0"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "670771e265a0b62dbd3c1fec2b865177eaf0bafd0ae49dd40a1c9fcd9a847a81",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014d45d1b0022c7387e42c5452ced561bdb8fd4b521",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d45d1b0022c7387e42c5452ced561bdb8fd4b521",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q63w3kqpzcuu8usk9g5kw64smmw8afdfpxmc2m0",
+                    "value": 453275
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022071312921800441903b2099e723add8702dd0f92ec11526ff87acf6967ec64cbd02203deabe7ed56d5daaa9a95c5a607b1ab705ff1c46bc6984a6dca120e63a91768601",
+                    "0257302ac8d9c4c8f9b1744f19bb432359326b9cc7bdddeeab9202749a6d92be58"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "0af82159eee2b69242f2ff032636e410b67ec1ace52e55fb0d20ed814cd64803",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001459e4d6bfefc6b45f955a69c4aeca26348e9d54ed",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 59e4d6bfefc6b45f955a69c4aeca26348e9d54ed",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qt8jdd0l0c669l926d8z2aj3xxj8f648dtyn7tc",
+                    "value": 439961
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022027540322e92c23c5513aa2587e7feb56a8ce82f879269d6b3cbd425634b44f8e022045572dee7262b02130bfe32d8aa8abbfaa64e101abfc819bba5380c78876692d01",
+                    "03fe02262d87f4a5289d3dd66e3d9a74cd49fa1cad0249284a7451896a827249a5"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "68cf9c784870a4f888f044755f7ce318557f652461db8ef887d279672f186018",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001454822b2d5d52597a78b630921cf439a41e32f2f9",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 54822b2d5d52597a78b630921cf439a41e32f2f9",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q2jpzkt2a2fvh579kxzfpeape5s0r9uhewhl5n4",
+                    "value": 227639
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402203ad511d6a8730748b8828bc38897d360451adf620ebdc1d229c08c097c80bef202202f50c793d95b5200cf2258e03896a3be7720df0eb3b8c810c86db74341a7e83e01",
+                    "0294992e9f4546e6e119741f908411ae531e9d1ff732d69b4dff8172aaf2a4b216"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "793f01dfdb19bf41f958fd917c16d9c4dd5d5e1a5c0434bfdb367212659d1b5b",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014f54edf8ae647b5300e2674523254e923d93d169f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 f54edf8ae647b5300e2674523254e923d93d169f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q748dlzhxg76nqr3xw3fry48fy0vn695lvhlkxv",
+                    "value": 227070
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402206e807ab616f4f2887ba703ae744d856142d9aca8128698419bbb67fb4fad8177022060fc65c7cd66baa88ad1e1d317a6edd5f6cb52fe8bff6e5405ffa1acf9d945d901",
+                    "02a0ad0167c6e9edf62677404d74d3b80ea276e47e758ffaa6ca17bd65ac79f7aa"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "00148a5c45ccfc29d209940d94525e2edb7743a1ad8a",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 8a5c45ccfc29d209940d94525e2edb7743a1ad8a",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1q3fwytn8u98fqn9qdj3f9utkmwap6rtv2ym33zm",
+                "value": 5500000
+            }
+        ],
+        "size": 1375,
+        "weight": 2605,
+        "fee": 691,
+        "status": {
+            "confirmed": false
+        },
+        "order": 1788986599,
+        "vsize": 651,
+        "adjustedVsize": 651.25,
+        "sigops": 9,
+        "feePerVsize": 1.0610364683301343,
+        "adjustedFeePerVsize": 1.0610364683301343,
+        "effectiveFeePerVsize": 1.0610364683301343,
+        "firstSeen": 1691163298,
+        "uid": 120494,
+        "inputs": [],
+        "position": {
+            "block": 7,
+            "vsize": 93780091.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    }
+]

backend/src/__tests__/api/test-data/transactions-rbfs.json

@@ -0,0 +1,121 @@
+[
+    {
+        "txid": "7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6",
+        "version": 1,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                    "value": 799995000
+                },
+                "scriptsig": "483045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "scriptsig_asm": "OP_PUSHBYTES_72 3045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a4c5058325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                "value": 799675549
+            }
+        ],
+        "size": 350,
+        "weight": 1400,
+        "fee": 9451,
+        "status": {
+            "confirmed": false
+        },
+        "order": 2798688215,
+        "vsize": 350,
+        "adjustedVsize": 350,
+        "sigops": 8,
+        "feePerVsize": 27.002857142857142,
+        "adjustedFeePerVsize": 27.002857142857142,
+        "effectiveFeePerVsize": 27.002857142857142,
+        "firstSeen": 1691218536,
+        "uid": 513598,
+        "inputs": [],
+        "position": {
+            "block": 0,
+            "vsize": 22166
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz",
+                    "value": 612917
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3045022100a0c23953ace5d022b7a6d45d1ae1730bf20a4d594bb5d4fa7aa80e4881b44d320220008f9b144805bb91995fc0f452a56e09f4ad16fa149d71ae9b5d57c742e8e2cc01",
+                    "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483649
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "0014199a98f9589364ffe5ef5bbae45ce5dfcbb873bd",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 199a98f9589364ffe5ef5bbae45ce5dfcbb873bd",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qrxdf372cjdj0le00twawgh89ml9msuaau62gk4",
+                "value": 611909
+            }
+        ],
+        "size": 192,
+        "weight": 438,
+        "fee": 1008,
+        "status": {
+            "confirmed": false
+        },
+        "bestDescendant": null,
+        "descendants": null,
+        "adjustedFeePerVsize": 10.2283,
+        "sigops": 1,
+        "adjustedVsize": 109.5
+    }
+]

backend/src/__tests__/api/test-data/transactions-replaced.json

@@ -0,0 +1,139 @@
+[
+    {
+        "txid": "008592364e21c1e3d62ba9538ac78a81779897b52100af5707ab063df98964f2",
+        "version": 1,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                    "value": 799995000
+                },
+                "scriptsig": "483045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "scriptsig_asm": "OP_PUSHBYTES_72 3045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a4c5058325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                "value": 799676250
+            }
+        ],
+        "size": 350,
+        "weight": 1400,
+        "fee": 8750,
+        "status": {
+            "confirmed": false
+        },
+        "order": 4066675193,
+        "vsize": 350,
+        "adjustedVsize": 350,
+        "sigops": 8,
+        "feePerVsize": 25,
+        "adjustedFeePerVsize": 25,
+        "effectiveFeePerVsize": 25,
+        "firstSeen": 1691218516,
+        "uid": 512584,
+        "inputs": [],
+        "position": {
+            "block": 0,
+            "vsize": 13846
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "b7981a624e4261c11f1246314d41e74be56af82eb557bcd054a5e0f94c023668",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz",
+                    "value": 612917
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204dd10f14afa41bc76d8278140ff1ec3d3f87f2c207bbb5418cc76dab30d7f6a402207877cc9c6a2c724b6ea7a1c24ac00022469f194fd1a4bd8030bbca1787d3f5f301",
+                    "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483648
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd",
+                "value": 344697
+            },
+            {
+                "scriptpubkey": "00144c2671336ca8761863b4c68d64d4672491fec1b9",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 4c2671336ca8761863b4c68d64d4672491fec1b9",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qfsn8zvmv4pmpsca5c6xkf4r8yjglasdesrawcx",
+                "value": 267636
+            }
+        ],
+        "size": 225,
+        "weight": 573,
+        "fee": 584,
+        "status": {
+            "confirmed": false
+        },
+        "order": 1748369996,
+        "vsize": 143,
+        "adjustedVsize": 143.25,
+        "sigops": 5,
+        "feePerVsize": 4.076788830715532,
+        "adjustedFeePerVsize": 4.076788830715532,
+        "effectiveFeePerVsize": 4.076788830715532,
+        "firstSeen": 1691222376,
+        "uid": 526515,
+        "inputs": [],
+        "position": {
+            "block": 7,
+            "vsize": 22021095.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    }
+]

backend/src/__tests__/config.test.ts

@@ -23,6 +23,7 @@ describe('Mempool Backend Config', () => {
         AUTOMATIC_BLOCK_REINDEXING: false,
         POLL_RATE_MS: 2000,
         CACHE_DIR: './cache',
+        CACHE_ENABLED: true,
         CLEAR_PROTECTION_MINUTES: 20,
         RECOMMENDED_FEE_PERCENTILE: 50,
         BLOCK_WEIGHT_UNITS: 4000000,
@@ -46,11 +47,17 @@ describe('Mempool Backend Config', () => {
         DISK_CACHE_BLOCK_INTERVAL: 6,
         MAX_PUSH_TX_SIZE_WEIGHT: 400000,
         ALLOW_UNREACHABLE: true,
+        PRICE_UPDATES_PER_HOUR: 1,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
 
-      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, RETRY_UNIX_SOCKET_AFTER: 30000 });
+      expect(config.ESPLORA).toStrictEqual({
+        REST_API_URL: 'http://127.0.0.1:3000',
+        UNIX_SOCKET_PATH: null,
+        RETRY_UNIX_SOCKET_AFTER: 30000,
+        FALLBACK: [],
+       });
 
       expect(config.CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',
@@ -100,11 +107,6 @@ describe('Mempool Backend Config', () => {
         PASSWORD: ''
       });
 
-      expect(config.PRICE_DATA_SERVER).toStrictEqual({
-        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
-        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
-      });
-
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
         MEMPOOL_API: 'https://mempool.space/api/v1',
         MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@@ -127,6 +129,16 @@ describe('Mempool Backend Config', () => {
         AUDIT_START_HEIGHT: 774000,
         SERVERS: []
       });
+
+      expect(config.MEMPOOL_SERVICES).toStrictEqual({
+        API: "",
+        ACCELERATIONS: false,
+      });
+
+      expect(config.REDIS).toStrictEqual({
+        ENABLED: false,
+        UNIX_SOCKET_PATH: ''
+      });
     });
   });
 
@@ -157,9 +169,11 @@ describe('Mempool Backend Config', () => {
 
       expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
 
-      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
-
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+
+      expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES);
+
+      expect(config.REDIS).toStrictEqual(fixture.REDIS);
     });
   });
 
@@ -172,41 +186,50 @@ describe('Mempool Backend Config', () => {
         for (const [key, value] of Object.entries(jsonObj)) {
           // We have a few cases where we can't follow the pattern
           if (root === 'MEMPOOL' && key === 'HTTP_PORT') {
-            console.log('skipping check for MEMPOOL_HTTP_PORT');
-            return;
-          }
-          switch (typeof value) {
-            case 'object': {
-              if (Array.isArray(value)) {
-                return;
-              } else {
-                parseJson(value, key);
-              }
-              break;
+            if (process.env.CI) {
+              console.log('skipping check for MEMPOOL_HTTP_PORT');
             }
-            default: {
+            continue;
+          }
+
+          if (root) {
               //The flattened string, i.e, __MEMPOOL_ENABLED__
               const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`;
 
               //The string used as the environment variable, i.e, MEMPOOL_ENABLED
               const envVarStr = `${root ? root : ''}_${key}`;
 
+              let defaultEntry;
               //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)}
-              const defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}';
-
-              console.log(`looking for ${defaultEntry} in the start.sh script`);
-              const re = new RegExp(defaultEntry);
-              expect(startSh).toMatch(re);
+              if (Array.isArray(value)) {
+                defaultEntry = `${replaceStr}=\${${envVarStr}:=[]}`;
+                if (process.env.CI) {
+                  console.log(`looking for ${defaultEntry} in the start.sh script`);
+                }
+                //Regex matching does not work with the array values
+                expect(startSh).toContain(defaultEntry);
+              } else {
+                 defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}';
+                 if (process.env.CI) {
+                  console.log(`looking for ${defaultEntry} in the start.sh script`);
+                }
+                const re = new RegExp(defaultEntry);
+                expect(startSh).toMatch(re);
+              }
 
               //The string that actually replaces the values in the config file
               const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json';
-              console.log(`looking for ${sedStr} in the start.sh script`);
+              if (process.env.CI) {
+                console.log(`looking for ${sedStr} in the start.sh script`);
+              }
               expect(startSh).toContain(sedStr);
-              break;
             }
+          else {
+            parseJson(value, key);
           }
         }
       }
+
       parseJson(fixture);
     });
   });

backend/src/__tests__/gbt/gbt-tests.ts

@@ -1,5 +1,5 @@
 import fs from 'fs';
-import { GbtGenerator, ThreadTransaction } from '../../../rust-gbt';
+import { GbtGenerator, ThreadTransaction } from 'rust-gbt';
 import path from 'path';
 
 const baseline = require('./test-data/target-template.json');
@@ -15,7 +15,7 @@ describe('Rust GBT', () => {
   test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => {
     const rustGbt = new GbtGenerator();
     const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer);
-    const result = await rustGbt.make(mempool, maxUid);
+    const result = await rustGbt.make(mempool, [], maxUid);
 
     const blocks: [string, number][][] = result.blocks.map(block => {
       return block.map(uid => [vectorUidMap.get(uid) || 'missing', uid]);

backend/src/api/audit.ts

@@ -6,16 +6,17 @@ import rbfCache from './rbf-cache';
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
-  auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
-   : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } {
+  auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false)
+   : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 };
+      return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
     const added: string[] = []; // present in mined block, not in template
     const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
-    const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement
+    const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
+    const accelerated: string[] = []; // prioritized by the mempool accelerator
     const isCensored = {}; // missing, without excuse
     const isDisplaced = {};
     let displacedWeight = 0;
@@ -28,6 +29,9 @@ class Audit {
     const now = Math.round((Date.now() / 1000));
     for (const tx of transactions) {
       inBlock[tx.txid] = tx;
+      if (mempool[tx.txid] && mempool[tx.txid].acceleration) {
+        accelerated.push(tx.txid);
+      }
     }
     // coinbase is always expected
     if (transactions[0]) {
@@ -36,8 +40,9 @@ class Audit {
     // look for transactions that were expected in the template, but missing from the mined block
     for (const txid of projectedBlocks[0].transactionIds) {
       if (!inBlock[txid]) {
-        if (rbfCache.isFullRbf(txid)) {
-          fullrbf.push(txid);
+        // allow missing transactions which either belong to a full rbf tree, or conflict with any transaction in the mined block
+        if (rbfCache.has(txid) && (rbfCache.isFullRbf(txid) || rbfCache.anyInSameTree(txid, (tx) => inBlock[tx.txid]))) {
+          rbf.push(txid);
         } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
           // tx is recent, may have reached the miner too late for inclusion
           fresh.push(txid);
@@ -98,8 +103,8 @@ class Audit {
       if (inTemplate[tx.txid]) {
         matches.push(tx.txid);
       } else {
-        if (rbfCache.isFullRbf(tx.txid)) {
-          fullrbf.push(tx.txid);
+        if (rbfCache.has(tx.txid)) {
+          rbf.push(tx.txid);
         } else if (!isDisplaced[tx.txid]) {
           added.push(tx.txid);
         }
@@ -147,7 +152,8 @@ class Audit {
       added,
       fresh,
       sigop: [],
-      fullrbf,
+      fullrbf: rbf,
+      accelerated,
       score,
       similarity,
     };

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -3,10 +3,13 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
+  $getAllMempoolTransactions(lastTxid: string);
   $getTransactionHex(txId: string): Promise<string>;
   $getBlockHeightTip(): Promise<number>;
   $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
@@ -20,6 +23,8 @@ export interface AbstractBitcoinApi {
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
   $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+
+  startHealthChecks(): void;
 }
 export interface BitcoinRpcCredentials {
   host: string;

backend/src/api/bitcoin/bitcoin-api.ts

@@ -5,6 +5,7 @@ import { IEsploraApi } from './esplora-api.interface';
 import blocks from '../blocks';
 import mempool from '../mempool';
 import { TransactionExtended } from '../../mempool.interfaces';
+import transactionUtils from '../transaction-utils';
 
 class BitcoinApi implements AbstractBitcoinApi {
   private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
@@ -59,9 +60,25 @@ class BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
-  $getTransactionHex(txId: string): Promise<string> {
-    return this.$getRawTransaction(txId, true)
-      .then((tx) => tx.hex || '');
+  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.');
+  }
+
+  $getAllMempoolTransactions(lastTxid: string): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.');
+
+  }
+
+  async $getTransactionHex(txId: string): Promise<string> {
+    const txInMempool = mempool.getMempool()[txId];
+    if (txInMempool && txInMempool.hex) {
+      return txInMempool.hex;
+    }
+
+    return this.bitcoindClient.getRawTransaction(txId, true)
+      .then((transaction: IBitcoinApi.Transaction) => {
+        return transaction.hex;
+      });
   }
 
   $getBlockHeightTip(): Promise<number> {
@@ -77,6 +94,10 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getTxsForBlock not supported by the Bitcoin RPC API.');
+  }
+
   $getRawBlock(hash: string): Promise<Buffer> {
     return this.bitcoindClient.getBlock(hash, 0)
       .then((raw: string) => Buffer.from(raw, "hex"));
@@ -201,7 +222,7 @@ class BitcoinApi implements AbstractBitcoinApi {
         scriptpubkey: vout.scriptPubKey.hex,
         scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
           : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
-        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
+        scriptpubkey_asm: vout.scriptPubKey.asm ? transactionUtils.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
         scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
       };
     });
@@ -211,7 +232,7 @@ class BitcoinApi implements AbstractBitcoinApi {
         is_coinbase: !!vin.coinbase,
         prevout: null,
         scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
-        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
+        scriptsig_asm: vin.scriptSig && transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) || '',
         sequence: vin.sequence,
         txid: vin.txid || '',
         vout: vin.vout || 0,
@@ -283,7 +304,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       }
       const innerTx = await this.$getRawTransaction(vin.txid, false, false);
       vin.prevout = innerTx.vout[vin.vout];
-      this.addInnerScriptsToVin(vin);
+      transactionUtils.addInnerScriptsToVin(vin);
     }
     return transaction;
   }
@@ -322,7 +343,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       }
       const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
       transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
-      this.addInnerScriptsToVin(transaction.vin[i]);
+      transactionUtils.addInnerScriptsToVin(transaction.vin[i]);
       totalIn += innerTx.vout[transaction.vin[i].vout].value;
     }
     if (lazyPrevouts && transaction.vin.length > 12) {
@@ -334,122 +355,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return transaction;
   }
 
-  private convertScriptSigAsm(hex: string): string {
-    const buf = Buffer.from(hex, 'hex');
-
-    const b: string[] = [];
-
-    let i = 0;
-    while (i < buf.length) {
-      const op = buf[i];
-      if (op >= 0x01 && op <= 0x4e) {
-        i++;
-        let push: number;
-        if (op === 0x4c) {
-          push = buf.readUInt8(i);
-          b.push('OP_PUSHDATA1');
-          i += 1;
-        } else if (op === 0x4d) {
-          push = buf.readUInt16LE(i);
-          b.push('OP_PUSHDATA2');
-          i += 2;
-        } else if (op === 0x4e) {
-          push = buf.readUInt32LE(i);
-          b.push('OP_PUSHDATA4');
-          i += 4;
-        } else {
-          push = op;
-          b.push('OP_PUSHBYTES_' + push);
-        }
-
-        const data = buf.slice(i, i + push);
-        if (data.length !== push) {
-          break;
-        }
-
-        b.push(data.toString('hex'));
-        i += data.length;
-      } else {
-        if (op === 0x00) {
-          b.push('OP_0');
-        } else if (op === 0x4f) {
-          b.push('OP_PUSHNUM_NEG1');
-        } else if (op === 0xb1) {
-          b.push('OP_CLTV');
-        } else if (op === 0xb2) {
-          b.push('OP_CSV');
-        } else if (op === 0xba) {
-          b.push('OP_CHECKSIGADD');
-        } else {
-          const opcode = bitcoinjs.script.toASM([ op ]);
-          if (opcode && op < 0xfd) {
-            if (/^OP_(\d+)$/.test(opcode)) {
-              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
-            } else {
-              b.push(opcode);
-            }
-          } else {
-            b.push('OP_RETURN_' + op);
-          }
-        }
-        i += 1;
-      }
-    }
-
-    return b.join(' ');
-  }
-
-  private addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
-    if (!vin.prevout) {
-      return;
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'p2sh') {
-      const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
-      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
-      if (vin.witness && vin.witness.length > 2) {
-        const witnessScript = vin.witness[vin.witness.length - 1];
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
-      }
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
-      const witnessScript = vin.witness[vin.witness.length - 1];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) {
-      const witnessScript = this.witnessToP2TRScript(vin.witness);
-      if (witnessScript !== null) {
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
-      }
-    }
-  }
-
-  /**
-   * This function must only be called when we know the witness we are parsing
-   * is a taproot witness.
-   * @param witness An array of hex strings that represents the witness stack of
-   *                the input.
-   * @returns null if the witness is not a script spend, and the hex string of
-   *          the script item if it is a script spend.
-   */
-  private witnessToP2TRScript(witness: string[]): string | null {
-    if (witness.length < 2) return null;
-    // Note: see BIP341 for parsing details of witness stack
-
-    // If there are at least two witness elements, and the first byte of the
-    // last element is 0x50, this last element is called annex a and
-    // is removed from the witness stack.
-    const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50';
-    // If there are at least two witness elements left, script path spending is used.
-    // Call the second-to-last stack element s, the script.
-    // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
-    if (hasAnnex && witness.length < 3) return null;
-    const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
-    return witness[positionOfScript];
-  }
-
+  public startHealthChecks(): void {};
 }
 
 export default BitcoinApi;

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler';
 import mempool from '../mempool';
 import feeApi from '../fee-api';
 import mempoolBlocks from '../mempool-blocks';
-import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
+import bitcoinApi from './bitcoin-api-factory';
 import { Common } from '../common';
 import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
@@ -214,6 +214,7 @@ class BitcoinRoutes {
           effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
           sigops: tx.sigops,
           adjustedVsize: tx.adjustedVsize,
+          acceleration: tx.acceleration
         });
         return;
       }
@@ -414,7 +415,7 @@ class BitcoinRoutes {
 
   private async getBlocks(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
         const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
         res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
         res.json(await blocks.$getBlocks(height, 15));
@@ -428,7 +429,7 @@ class BitcoinRoutes {
 
   private async getBlocksByBulk(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
         return res.status(404).send(`This API is only available for Bitcoin networks`);
       }
       if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
@@ -483,7 +484,7 @@ class BitcoinRoutes {
           returnBlocks.push(localBlock);
           nextHash = localBlock.previousblockhash;
         } else {
-          const block = await bitcoinCoreApi.$getBlock(nextHash);
+          const block = await bitcoinApi.$getBlock(nextHash);
           returnBlocks.push(block);
           nextHash = block.previousblockhash;
         }
@@ -576,7 +577,7 @@ class BitcoinRoutes {
     }
 
     try {
-      const addressData = await bitcoinApi.$getScriptHash(req.params.address);
+      const addressData = await bitcoinApi.$getScriptHash(req.params.scripthash);
       res.json(addressData);
     } catch (e) {
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
@@ -597,7 +598,7 @@ class BitcoinRoutes {
       if (req.query.after_txid && typeof req.query.after_txid === 'string') {
         lastTxId = req.query.after_txid;
       }
-      const transactions = await bitcoinApi.$getScriptHashTransactions(req.params.address, lastTxId);
+      const transactions = await bitcoinApi.$getScriptHashTransactions(req.params.scripthash, lastTxId);
       res.json(transactions);
     } catch (e) {
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {

backend/src/api/bitcoin/esplora-api.ts

@@ -1,104 +1,260 @@
 import config from '../../config';
-import axios, { AxiosRequestConfig } from 'axios';
+import axios, { AxiosResponse } from 'axios';
 import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import logger from '../../logger';
 
-const axiosConnection = axios.create({
-  httpAgent: new http.Agent({ keepAlive: true, })
-});
+interface FailoverHost {
+  host: string,
+  rtts: number[],
+  rtt: number
+  failures: number,
+  socket?: boolean,
+  outOfSync?: boolean,
+  unreachable?: boolean,
+  preferred?: boolean,
+}
 
-class ElectrsApi implements AbstractBitcoinApi {
-  private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
-    socketPath: config.ESPLORA.UNIX_SOCKET_PATH,
-    timeout: 10000,
-  } : {
-    timeout: 10000,
-  };
-  private axiosConfigTcpSocketOnly: AxiosRequestConfig = {
-    timeout: 10000,
-  };
-
-  unixSocketRetryTimeout;
-  activeAxiosConfig;
+class FailoverRouter {
+  activeHost: FailoverHost;
+  fallbackHost: FailoverHost;
+  hosts: FailoverHost[];
+  multihost: boolean;
+  pollInterval: number = 60000;
+  pollTimer: NodeJS.Timeout | null = null;
+  pollConnection = axios.create();
+  requestConnection = axios.create({
+    httpAgent: new http.Agent({ keepAlive: true })
+  });
 
   constructor() {
-    this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
+    // setup list of hosts
+    this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => {
+      return {
+        host: domain,
+        rtts: [],
+        rtt: Infinity,
+        failures: 0,
+      };
+    });
+    this.activeHost = {
+      host: config.ESPLORA.UNIX_SOCKET_PATH || config.ESPLORA.REST_API_URL,
+      rtts: [],
+      rtt: 0,
+      failures: 0,
+      socket: !!config.ESPLORA.UNIX_SOCKET_PATH,
+      preferred: true,
+    };
+    this.fallbackHost = this.activeHost;
+    this.hosts.unshift(this.activeHost);
+    this.multihost = this.hosts.length > 1;
   }
 
-  fallbackToTcpSocket() {
-    if (!this.unixSocketRetryTimeout) {
-      logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`);
-      // Retry the unix socket after a few seconds
-      this.unixSocketRetryTimeout = setTimeout(() => {
-        logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`);
-        this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
-        this.unixSocketRetryTimeout = undefined;
-      }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER);
+  public startHealthChecks(): void {
+    // use axios interceptors to measure request rtt
+    this.pollConnection.interceptors.request.use((config) => {
+      config['meta'] = { startTime: Date.now() };
+      return config;
+    });
+    this.pollConnection.interceptors.response.use((response) => {
+      response.config['meta'].rtt = Date.now() - response.config['meta'].startTime;
+      return response;
+    });
+
+    if (this.multihost) {
+      this.pollHosts();
+    }
+  }
+
+  // start polling hosts to measure availability & rtt
+  private async pollHosts(): Promise<void> {
+    if (this.pollTimer) {
+      clearTimeout(this.pollTimer);
     }
 
-    // Use the TCP socket (reach a different esplora instance through nginx)
-    this.activeAxiosConfig = this.axiosConfigTcpSocketOnly;
+    const results = await Promise.allSettled(this.hosts.map(async (host) => {
+      if (host.socket) {
+        return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: 2000 });
+      } else {
+        return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: 2000 });
+      }
+    }));
+    const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0);
+
+    // update rtts & sync status
+    for (let i = 0; i < results.length; i++) {
+      const host = this.hosts[i];
+      const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult<AxiosResponse<number, any>>).value : null;
+      if (result) {
+        const height = result.data;
+        const rtt = result.config['meta'].rtt;
+        host.rtts.unshift(rtt);
+        host.rtts.slice(0, 5);
+        host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length;
+        if (height == null || isNaN(height) || (maxHeight - height > 2)) {
+          host.outOfSync = true;
+        } else {
+          host.outOfSync = false;
+        }
+        host.unreachable = false;
+      } else {
+        host.unreachable = true;
+      }
+    }
+
+    this.sortHosts();
+
+    logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg rtt ' + Math.round(host.rtt).toString().padStart(5, ' ') + ' | reachable? ' + (!host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + (!host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`);
+
+    // switch if the current host is out of sync or significantly slower than the next best alternative
+    if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== this.hosts[0] && this.hosts[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) {
+      if (this.activeHost.unreachable) {
+        logger.warn(`Unable to reach ${this.activeHost.host}, failing over to next best alternative`);
+      } else if (this.activeHost.outOfSync) {
+        logger.warn(`${this.activeHost.host} has fallen behind, failing over to next best alternative`);
+      } else {
+        logger.debug(`${this.activeHost.host} is no longer the best esplora host`);
+      }
+      this.electHost();
+    }
+
+    this.pollTimer = setTimeout(() => { this.pollHosts(); }, this.pollInterval);
   }
 
-  $queryWrapper<T>(url, responseType = 'json'): Promise<T> {
-    return axiosConnection.get<T>(url, { ...this.activeAxiosConfig, responseType: responseType })
-      .then((response) => response.data)
+  // sort hosts by connection quality, and update default fallback
+  private sortHosts(): void {
+    // sort by connection quality
+    this.hosts.sort((a, b) => {
+      if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) {
+        if  (a.preferred === b.preferred) {
+          // lower rtt is best
+          return a.rtt - b.rtt;
+        } else { // unless we have a preferred host
+          return a.preferred ? -1 : 1;
+        }
+      } else { // or the host is out of sync
+        return (a.unreachable || a.outOfSync) ? 1 : -1;
+      }
+    });
+    if (this.hosts.length > 1 && this.hosts[0] === this.activeHost) {
+      this.fallbackHost = this.hosts[1];
+    } else {
+      this.fallbackHost = this.hosts[0];
+    }
+  }
+
+  // depose the active host and choose the next best replacement
+  private electHost(): void {
+    this.activeHost.outOfSync = true;
+    this.activeHost.failures = 0;
+    this.sortHosts();
+    this.activeHost = this.hosts[0];
+    logger.warn(`Switching esplora host to ${this.activeHost.host}`);
+  }
+
+  private addFailure(host: FailoverHost): FailoverHost {
+    host.failures++;
+    if (host.failures > 5 && this.multihost) {
+      logger.warn(`Too many esplora failures on ${this.activeHost.host}, falling back to next best alternative`);
+      this.electHost();
+      return this.activeHost;
+    } else {
+      return this.fallbackHost;
+    }
+  }
+
+  private async $query<T>(method: 'get'| 'post', path, data: any, responseType = 'json', host = this.activeHost, retry: boolean = true): Promise<T> {
+    let axiosConfig;
+    let url;
+    if (host.socket) {
+      axiosConfig = { socketPath: host.host, timeout: 10000, responseType };
+      url = path;
+    } else {
+      axiosConfig = { timeout: 10000, responseType };
+      url = host.host + path;
+    }
+    return (method === 'post'
+        ? this.requestConnection.post<T>(url, data, axiosConfig)
+        : this.requestConnection.get<T>(url, axiosConfig)
+    ).then((response) => { host.failures = Math.max(0, host.failures - 1); return response.data; })
       .catch((e) => {
-        if (e?.code === 'ECONNREFUSED') {
-          this.fallbackToTcpSocket();
+        let fallbackHost = this.fallbackHost;
+        if (e?.response?.status !== 404) {
+          logger.warn(`esplora request failed ${e?.response?.status || 500} ${host.host}${path}`);
+          fallbackHost = this.addFailure(host);
+        }
+        if (retry && e?.code === 'ECONNREFUSED' && this.multihost) {
           // Retry immediately
-          return axiosConnection.get<T>(url, this.activeAxiosConfig)
-            .then((response) => response.data)
-            .catch((e) => {
-              logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`);
-              throw e;
-            });
+          return this.$query(method, path, data, responseType, fallbackHost, false);
         } else {
           throw e;
         }
       });
   }
 
+  public async $get<T>(path, responseType = 'json'): Promise<T> {
+    return this.$query<T>('get', path, null, responseType);
+  }
+
+  public async $post<T>(path, data: any, responseType = 'json'): Promise<T> {
+    return this.$query<T>('post', path, data, responseType);
+  }
+}
+
+class ElectrsApi implements AbstractBitcoinApi {
+  private failoverRouter = new FailoverRouter();
+
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return this.$queryWrapper<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids');
+    return this.failoverRouter.$get<IEsploraApi.Transaction['txid'][]>('/mempool/txids');
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return this.$queryWrapper<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId);
+    return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txId);
+  }
+
+  async $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/mempool/txs', txids, 'json');
+  }
+
+  async $getAllMempoolTransactions(lastSeenTxid?: string): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''));
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex');
+    return this.failoverRouter.$get<string>('/tx/' + txId + '/hex');
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return this.$queryWrapper<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height');
+    return this.failoverRouter.$get<number>('/blocks/tip/height');
   }
 
   $getBlockHashTip(): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash');
+    return this.failoverRouter.$get<string>('/blocks/tip/hash');
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return this.$queryWrapper<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids');
+    return this.failoverRouter.$get<string[]>('/block/' + hash + '/txids');
+  }
+
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/block/' + hash + '/txs');
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height);
+    return this.failoverRouter.$get<string>('/block-height/' + height);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header');
+    return this.failoverRouter.$get<string>('/block/' + hash + '/header');
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return this.$queryWrapper<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash);
+    return this.failoverRouter.$get<IEsploraApi.Block>('/block/' + hash);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return this.$queryWrapper<any>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer')
+    return this.failoverRouter.$get<any>('/block/' + hash + '/raw', 'arraybuffer')
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -127,11 +283,11 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return this.$queryWrapper<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout);
+    return this.failoverRouter.$get<IEsploraApi.Outspend>('/tx/' + txId + '/outspend/' + vout);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return this.$queryWrapper<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends');
+    return this.failoverRouter.$get<IEsploraApi.Outspend[]>('/tx/' + txId + '/outspends');
   }
 
   async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
@@ -142,6 +298,10 @@ class ElectrsApi implements AbstractBitcoinApi {
     }
     return outspends;
   }
+
+  public startHealthChecks(): void {
+    this.failoverRouter.startHealthChecks();
+  }
 }
 
 export default ElectrsApi;

backend/src/api/blocks.ts

@@ -26,12 +26,15 @@ import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
 import chainTips from './chain-tips';
 import websocketHandler from './websocket-handler';
+import redisCache from './redis-cache';
+import rbfCache from './rbf-cache';
+import { calcBitsDifference } from './difficulty-adjustment';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
   private blockSummaries: BlockSummary[] = [];
   private currentBlockHeight = 0;
-  private currentDifficulty = 0;
+  private currentBits = 0;
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
@@ -70,6 +73,9 @@ class Blocks {
    * @param blockHash
    * @param blockHeight
    * @param onlyCoinbase - Set to true if you only need the coinbase transaction
+   * @param txIds - optional ordered list of transaction ids if already known
+   * @param quiet - don't print non-essential logs
+   * @param addMempoolData - calculate sigops etc
    * @returns Promise<TransactionExtended[]>
    */
   private async $getTransactionsExtended(
@@ -80,62 +86,98 @@ class Blocks {
     quiet: boolean = false,
     addMempoolData: boolean = false,
   ): Promise<TransactionExtended[]> {
-    const transactions: TransactionExtended[] = [];
+    const isEsplora = config.MEMPOOL.BACKEND === 'esplora';
+    const transactionMap: { [txid: string]: TransactionExtended } = {};
+
     if (!txIds) {
       txIds = await bitcoinApi.$getTxIdsForBlock(blockHash);
     }
 
     const mempool = memPool.getMempool();
-    let transactionsFound = 0;
-    let transactionsFetched = 0;
+    let foundInMempool = 0;
+    let totalFound = 0;
 
-    for (let i = 0; i < txIds.length; i++) {
-      if (mempool[txIds[i]]) {
-        // We update blocks before the mempool (index.ts), therefore we can
-        // optimize here by directly fetching txs in the "outdated" mempool
-        transactions.push(mempool[txIds[i]]);
-        transactionsFound++;
-      } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
-        // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
-        if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
-          logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
-        }
-        try {
-          const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, false, addMempoolData);
-          transactions.push(tx);
-          transactionsFetched++;
-        } catch (e) {
-          try {
-            if (config.MEMPOOL.BACKEND === 'esplora') {
-              // Try again with core
-              const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true, addMempoolData);
-              transactions.push(tx);
-              transactionsFetched++;
-            } else {
-              throw e;
-            }
-          } catch (e) {
-            if (i === 0) {
-              const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
-              logger.err(msg);
-              throw new Error(msg);
-            } else {
-              logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
-            }
-          }
+    // Copy existing transactions from the mempool
+    if (!onlyCoinbase) {
+      for (const txid of txIds) {
+        if (mempool[txid]) {
+          transactionMap[txid] = mempool[txid];
+          foundInMempool++;
+          totalFound++;
         }
       }
+    }
 
-      if (onlyCoinbase === true) {
-        break; // Fetch the first transaction and exit
+    if (onlyCoinbase) {
+      try {
+        const coinbase = await transactionUtils.$getTransactionExtendedRetry(txIds[0], false, false, false, addMempoolData);
+        if (coinbase && coinbase.vin[0].is_coinbase) {
+          return [coinbase];
+        } else {
+          const msg = `Expected a coinbase tx, but the backend API returned something else`;
+          logger.err(msg);
+          throw new Error(msg);
+        }
+      } catch (e) {
+        const msg = `Cannot fetch coinbase tx ${txIds[0]}. Reason: ` + (e instanceof Error ? e.message : e);
+        logger.err(msg);
+        throw new Error(msg);
+      }
+    }
+
+    // Fetch remaining txs in bulk
+    if (isEsplora && (txIds.length - totalFound > 500)) {
+      try {
+        const rawTransactions = await bitcoinApi.$getTxsForBlock(blockHash);
+        for (const tx of rawTransactions) {
+          if (!transactionMap[tx.txid]) {
+            transactionMap[tx.txid] = addMempoolData ? transactionUtils.extendMempoolTransaction(tx) : transactionUtils.extendTransaction(tx);
+            totalFound++;
+          }
+        }
+      } catch (e) {
+        logger.err(`Cannot fetch bulk txs for block ${blockHash}. Reason: ` + (e instanceof Error ? e.message : e));
+      }
+    }
+
+    // Fetch remaining txs individually
+    for (const txid of txIds.filter(txid => !transactionMap[txid])) {
+      if (!quiet && (totalFound % (Math.round((txIds.length) / 10)) === 0 || totalFound + 1 === txIds.length)) { // Avoid log spam
+        logger.debug(`Indexing tx ${totalFound + 1} of ${txIds.length} in block #${blockHeight}`);
+      }
+      try {
+        const tx = await transactionUtils.$getTransactionExtendedRetry(txid, false, false, false, addMempoolData);
+        transactionMap[txid] = tx;
+        totalFound++;
+      } catch (e) {
+        const msg = `Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e);
+        logger.err(msg);
+        throw new Error(msg);
       }
     }
 
     if (!quiet) {
-      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+      logger.debug(`${foundInMempool} of ${txIds.length} found in mempool. ${totalFound - foundInMempool} fetched through backend service.`);
     }
 
-    return transactions;
+    // Require the first transaction to be a coinbase
+    const coinbase = transactionMap[txIds[0]];
+    if (!coinbase || !coinbase.vin[0].is_coinbase) {
+      const msg = `Expected first tx in a block to be a coinbase, but found something else`;
+      logger.err(msg);
+      throw new Error(msg);
+    }
+
+    // Require all transactions to be present
+    // (we should have thrown an error already if a tx request failed)
+    if (txIds.some(txid => !transactionMap[txid])) {
+      const msg = `Failed to fetch ${txIds.length - totalFound} transactions from block`;
+      logger.err(msg);
+      throw new Error(msg);
+    }
+
+    // Return list of transactions, preserving block order
+    return txIds.map(txid => transactionMap[txid]);
   }
 
   /**
@@ -263,7 +305,7 @@ class Blocks {
       extras.totalInputAmt = null;
     }
 
-    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
       if (coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(coinbaseTx);
@@ -378,8 +420,8 @@ class Blocks {
       let newlyIndexed = 0;
       let totalIndexed = indexedBlockSummariesHashesArray.length;
       let indexedThisRun = 0;
-      let timer = new Date().getTime() / 1000;
-      const startedAt = new Date().getTime() / 1000;
+      let timer = Date.now() / 1000;
+      const startedAt = Date.now() / 1000;
 
       for (const block of indexedBlocks) {
         if (indexedBlockSummariesHashes[block.hash] === true) {
@@ -387,17 +429,24 @@ class Blocks {
         }
 
         // Logging
-        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        const elapsedSeconds = (Date.now() / 1000) - timer;
         if (elapsedSeconds > 5) {
-          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-          const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
+          const runningFor = (Date.now() / 1000) - startedAt;
+          const blockPerSeconds = indexedThisRun / elapsedSeconds;
           const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
-          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
-          timer = new Date().getTime() / 1000;
+          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining);
+          timer = Date.now() / 1000;
           indexedThisRun = 0;
         }
 
-        await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
+
+        if (config.MEMPOOL.BACKEND === 'esplora') {
+          const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
+          const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
+          await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
+        } else {
+          await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
+        }
 
         // Logging
         indexedThisRun++;
@@ -436,18 +485,18 @@ class Blocks {
       // Logging
       let count = 0;
       let countThisRun = 0;
-      let timer = new Date().getTime() / 1000;
-      const startedAt = new Date().getTime() / 1000;
+      let timer = Date.now() / 1000;
+      const startedAt = Date.now() / 1000;
       for (const height of unindexedBlockHeights) {
         // Logging
         const hash = await bitcoinApi.$getBlockHash(height);
-        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        const elapsedSeconds = (Date.now() / 1000) - timer;
         if (elapsedSeconds > 5) {
-          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-          const blockPerSeconds = (countThisRun / elapsedSeconds);
+          const runningFor = (Date.now() / 1000) - startedAt;
+          const blockPerSeconds = countThisRun / elapsedSeconds;
           const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
-          logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`);
-          timer = new Date().getTime() / 1000;
+          logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`);
+          timer = Date.now() / 1000;
           countThisRun = 0;
         }
 
@@ -526,8 +575,8 @@ class Blocks {
       let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
       let indexedThisRun = 0;
       let newlyIndexed = 0;
-      const startedAt = new Date().getTime() / 1000;
-      let timer = new Date().getTime() / 1000;
+      const startedAt = Date.now() / 1000;
+      let timer = Date.now() / 1000;
 
       while (currentBlockHeight >= lastBlockToIndex) {
         const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
@@ -547,18 +596,18 @@ class Blocks {
           }
           ++indexedThisRun;
           ++totalIndexed;
-          const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+          const elapsedSeconds = (Date.now() / 1000) - timer;
           if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
-            const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-            const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
+            const runningFor = (Date.now() / 1000) - startedAt;
+            const blockPerSeconds = indexedThisRun / elapsedSeconds;
             const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
-            timer = new Date().getTime() / 1000;
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining);
+            timer = Date.now() / 1000;
             indexedThisRun = 0;
             loadingIndicators.setProgress('block-indexing', progress, false);
           }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
-          const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+          const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
           const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true);
           const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -615,17 +664,21 @@ class Blocks {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
         this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment');
-        const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+        const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
         this.updateTimerProgress(timer, 'got block for initial difficulty adjustment');
         this.lastDifficultyAdjustmentTime = block.timestamp;
-        this.currentDifficulty = block.difficulty;
+        this.currentBits = block.bits;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
           this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
-          const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
+          const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
           this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
-          this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
+          if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
+            this.previousDifficultyRetarget = NaN;
+          } else {
+            this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits);
+          }
           logger.debug(`Initial difficulty adjustment data set.`);
         }
       } else {
@@ -649,14 +702,14 @@ class Blocks {
       const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = verboseBlock.tx.map(tx => tx.txid);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[];
-      if (config.MEMPOOL.BACKEND !== 'esplora') {
-        // fill in missing transaction fee data from verboseBlock
-        for (let i = 0; i < transactions.length; i++) {
-          if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) {
-            transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000;
-          }
+
+      // fill in missing transaction fee data from verboseBlock
+      for (let i = 0; i < transactions.length; i++) {
+        if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) {
+          transactions[i].fee = (verboseBlock.tx[i].fee * 100_000_000) || 0;
         }
       }
+
       const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
@@ -734,18 +787,33 @@ class Blocks {
 
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
+          let adjustment;
+          if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
+            adjustment = NaN;
+          } else {
+            adjustment = Math.round(
+              // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio.
+              // Instead of actually doing /100, just reduce the multiplier.
+              (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000
+            ) / 1000000; // Remove float point noise
+          }
+
           await DifficultyAdjustmentsRepository.$saveAdjustments({
             time: block.timestamp,
             height: block.height,
             difficulty: block.difficulty,
-            adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
+            adjustment,
           });
           this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
         }
 
-        this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
+        if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
+          this.previousDifficultyRetarget = NaN;
+        } else {
+          this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits);
+        }
         this.lastDifficultyAdjustmentTime = block.timestamp;
-        this.currentDifficulty = block.difficulty;
+        this.currentBits = block.bits;
       }
 
       // wait for pending async callbacks to finish
@@ -765,10 +833,18 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
+      if (config.MEMPOOL.CACHE_ENABLED && !memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
         diskCache.$saveCacheToDisk();
       }
 
+      // Update Redis cache
+      if (config.REDIS.ENABLED) {
+        await redisCache.$updateBlocks(this.blocks);
+        await redisCache.$updateBlockSummaries(this.blockSummaries);
+        await redisCache.$removeTransactions(txIds);
+        await rbfCache.updateCache();
+      }
+
       handledBlocks++;
     }
 
@@ -813,7 +889,7 @@ class Blocks {
     }
 
     const blockHash = await bitcoinApi.$getBlockHash(height);
-    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
     const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -825,7 +901,7 @@ class Blocks {
   }
 
   public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
-    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
+    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
     const transactions = await this.$getTransactionsExtended(hash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -845,12 +921,12 @@ class Blocks {
     }
 
     // Not Bitcoin network, return the block as it from the bitcoin backend
-    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return await bitcoinCoreApi.$getBlock(hash);
     }
 
     // Bitcoin network, add our custom data on top
-    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
+    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
     if (block.stale) {
       return await this.$indexStaleBlock(hash);
     } else {
@@ -885,7 +961,7 @@ class Blocks {
         transactions: cpfpSummary.transactions.map(tx => {
           return {
             txid: tx.txid,
-            fee: tx.fee,
+            fee: tx.fee || 0,
             vsize: tx.vsize,
             value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)),
             rate: tx.effectiveFeePerVsize
@@ -893,10 +969,15 @@ class Blocks {
         }),
       };
     } else {
-      // Call Core RPC
-      const block = await bitcoinClient.getBlock(hash, 2);
-      summary = this.summarizeBlock(block);
-      height = block.height;
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
+        summary = this.summarizeBlockTransactions(hash, txs);
+      } else {
+        // Call Core RPC
+        const block = await bitcoinClient.getBlock(hash, 2);
+        summary = this.summarizeBlock(block);
+        height = block.height;
+      }
     }
     if (height == null) {
       const block = await bitcoinApi.$getBlock(hash);
@@ -1019,8 +1100,17 @@ class Blocks {
       if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
         cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
         if (cleanBlock.fee_amt_percentiles === null) {
-          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
-          const summary = this.summarizeBlock(block);
+
+          let summary;
+          if (config.MEMPOOL.BACKEND === 'esplora') {
+            const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
+            summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
+          } else {
+            // Call Core RPC
+            const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
+            summary = this.summarizeBlock(block);
+          }
+
           await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
           cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
         }
@@ -1061,7 +1151,7 @@ class Blocks {
   }
 
   public async $getBlockAuditSummary(hash: string): Promise<any> {
-    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       return BlocksAuditsRepository.$getBlockAudit(hash);
     } else {
       return null;
@@ -1080,19 +1170,29 @@ class Blocks {
     return this.currentBlockHeight;
   }
 
-  public async $indexCPFP(hash: string, height: number): Promise<void> {
-    const block = await bitcoinClient.getBlock(hash, 2);
-    const transactions = block.tx.map(tx => {
-      tx.fee *= 100_000_000;
-      return tx;
-    });
+  public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary> {
+    let transactions = txs;
+    if (!transactions) {
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
+      }
+      if (!transactions) {
+        const block = await bitcoinClient.getBlock(hash, 2);
+        transactions = block.tx.map(tx => {
+          tx.fee *= 100_000_000;
+          return tx;
+        });
+      }
+    }
 
-    const summary = Common.calculateCpfp(height, transactions);
+    const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]);
 
     await this.$saveCpfp(hash, height, summary);
 
     const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
     await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
+
+    return summary;
   }
 
   public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {

backend/src/api/common.ts

@@ -59,10 +59,12 @@ export class Common {
     return arr;
   }
 
-  static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended[] } {
+  static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: MempoolTransactionExtended[] } {
     const matches: { [txid: string]: MempoolTransactionExtended[] } = {};
-    added
-      .forEach((addedTx) => {
+
+    // For small N, a naive nested loop is extremely fast, but it doesn't scale
+    if (added.length < 1000 && deleted.length < 50 && !forceScalable) {
+      added.forEach((addedTx) => {
         const foundMatches = deleted.filter((deletedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
           return addedTx.fee > deletedTx.fee
@@ -73,9 +75,40 @@ export class Common {
               addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
         if (foundMatches?.length) {
-          matches[addedTx.txid] = foundMatches;
+          matches[addedTx.txid] = [...new Set(foundMatches)];
         }
       });
+    } else {
+      // for large N, build a lookup table of prevouts we can check in ~constant time
+      const deletedSpendMap: { [txid: string]: { [vout: number]: MempoolTransactionExtended } } = {};
+      for (const tx of deleted) {
+        for (const vin of tx.vin) {
+          if (!deletedSpendMap[vin.txid]) {
+            deletedSpendMap[vin.txid] = {};
+          }
+          deletedSpendMap[vin.txid][vin.vout] = tx;
+        }
+      }
+
+      for (const addedTx of added) {
+        const foundMatches = new Set<MempoolTransactionExtended>();
+        for (const vin of addedTx.vin) {
+          const deletedTx = deletedSpendMap[vin.txid]?.[vin.vout];
+          if (deletedTx && deletedTx.txid !== addedTx.txid
+              // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
+              && addedTx.fee > deletedTx.fee
+              // The new transaction must pay more fee per kB than the replaced tx.
+              && addedTx.adjustedFeePerVsize > deletedTx.adjustedFeePerVsize
+          ) {
+            foundMatches.add(deletedTx);
+          }
+          if (foundMatches.size) {
+            matches[addedTx.txid] = [...foundMatches];
+          }
+        }
+      }
+    }
+
     return matches;
   }
 
@@ -108,9 +141,10 @@ export class Common {
   static stripTransaction(tx: TransactionExtended): TransactionStripped {
     return {
       txid: tx.txid,
-      fee: tx.fee,
+      fee: tx.fee || 0,
       vsize: tx.weight / 4,
       value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0),
+      acc: tx.acceleration || undefined,
       rate: tx.effectiveFeePerVsize,
     };
   }
@@ -239,7 +273,7 @@ export class Common {
 
   static indexingEnabled(): boolean {
     return (
-      ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
+      ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
       config.DATABASE.ENABLED === true &&
       config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
     );
@@ -460,7 +494,7 @@ export class Common {
     };
   }
 
-  static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats {
+  static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
     const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
 
     let weightCount = 0;

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 64;
+  private static currentVersion = 65;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -104,7 +104,7 @@ class DatabaseMigration {
   private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
     await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
 
-    const isBitcoin = ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK);
+    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
 
     await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
     await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
@@ -512,7 +512,7 @@ class DatabaseMigration {
       await this.updateToSchemaVersion(58);
     }
 
-    if (databaseSchemaVersion < 59 && ['testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+    if (databaseSchemaVersion < 59 && (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet')) {
       // https://github.com/mempool/mempool/issues/3360
       await this.$executeQuery(`TRUNCATE prices`);
     }
@@ -548,6 +548,11 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
       await this.updateToSchemaVersion(64);
     }
+
+    if (databaseSchemaVersion < 65 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(65);
+    }
   }
 
   /**
@@ -656,7 +661,7 @@ class DatabaseMigration {
    */
   private getMigrationQueriesFromVersion(version: number): string[] {
     const queries: string[] = [];
-    const isBitcoin = ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK);
+    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
 
     if (version < 1) {
       if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {

backend/src/api/difficulty-adjustment.ts

@@ -16,6 +16,68 @@ export interface DifficultyAdjustment {
   expectedBlocks: number;         // Block count
 }
 
+/**
+ * Calculate the difficulty increase/decrease by using the `bits` integer contained in two
+ * block headers.
+ *
+ * Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code
+ * assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an
+ * error if an exponent difference of 2 or more is seen.
+ *
+ * @param {number} oldBits The 32 bit `bits` integer from a block header.
+ * @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period.
+ * @returns {number} A floating point decimal of the difficulty change from old to new.
+ *          (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty)
+ */
+export function calcBitsDifference(oldBits: number, newBits: number): number {
+  // Must be
+  // - integer
+  // - highest exponent is 0x20, so max value (as integer) is 0x207fffff
+  // - min value is 1 (exponent = 0)
+  // - highest bit of the number-part is +- sign, it must not be 1
+  const verifyBits = (bits: number): void => {
+    if (
+      Math.floor(bits) !== bits ||
+      bits > 0x207fffff ||
+      bits < 1 ||
+      (bits & 0x00800000) !== 0 ||
+      (bits & 0x007fffff) === 0
+    ) {
+      throw new Error('Invalid bits');
+    }
+  };
+  verifyBits(oldBits);
+  verifyBits(newBits);
+
+  // No need to mask exponents because we checked the bounds above
+  const oldExp = oldBits >> 24;
+  const newExp = newBits >> 24;
+  const oldNum = oldBits & 0x007fffff;
+  const newNum = newBits & 0x007fffff;
+  // The diff can only possibly be 1, 0, -1
+  // (because maximum difficulty change is x4 or /4 (2 bits up or down))
+  let result: number;
+  switch (newExp - oldExp) {
+    // New less than old, target lowered, difficulty increased
+    case -1:
+      result = ((oldNum << 8) * 100) / newNum - 100;
+      break;
+    // Same exponent, compare numbers as is.
+    case 0:
+      result = (oldNum * 100) / newNum - 100;
+      break;
+    // Old less than new, target raised, difficulty decreased
+    case 1:
+      result = (oldNum * 100) / (newNum << 8) - 100;
+      break;
+    default:
+      throw new Error('Impossible exponent difference');
+  }
+
+  // Min/Max values
+  return result > 300 ? 300 : result < -75 ? -75 : result;
+}
+
 export function calcDifficultyAdjustment(
   DATime: number,
   nowSeconds: number,

backend/src/api/disk-cache.ts

@@ -29,7 +29,7 @@ class DiskCache {
   };
 
   constructor() {
-    if (!cluster.isPrimary) {
+    if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
       return;
     }
     process.on('SIGINT', (e) => {
@@ -39,7 +39,7 @@ class DiskCache {
   }
 
   async $saveCacheToDisk(sync: boolean = false): Promise<void> {
-    if (!cluster.isPrimary) {
+    if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
       return;
     }
     if (this.isWritingCache) {
@@ -175,10 +175,11 @@ class DiskCache {
   }
 
   async $loadMempoolCache(): Promise<void> {
-    if (!fs.existsSync(DiskCache.FILE_NAME)) {
+    if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
     try {
+      const start = Date.now();
       let data: any = {};
       const cacheData = fs.readFileSync(DiskCache.FILE_NAME, 'utf8');
       if (cacheData) {
@@ -220,6 +221,8 @@ class DiskCache {
         }
       }
 
+      logger.info(`Loaded mempool from disk cache in ${Date.now() - start} ms`);
+
       await memPool.$setMempool(data.mempool);
       if (!this.ignoreBlocksCache) {
         blocks.setBlocks(data.blocks);

backend/src/api/mempool-blocks.ts

@@ -1,10 +1,11 @@
-import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt';
+import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
 import logger from '../logger';
-import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces';
+import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces';
 import { Common, OnlineFeeStatsCalculator } from './common';
 import config from '../config';
 import { Worker } from 'worker_threads';
 import path from 'path';
+import mempool from './mempool';
 
 const MAX_UINT32 = Math.pow(2, 32) - 1;
 
@@ -170,7 +171,7 @@ class MempoolBlocks {
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
       let removed: string[] = [];
-      const changed: { txid: string, rate: number | undefined }[] = [];
+      const changed: { txid: string, rate: number | undefined, acc: boolean | undefined }[] = [];
       if (mempoolBlocks[i] && !prevBlocks[i]) {
         added = mempoolBlocks[i].transactions;
       } else if (!mempoolBlocks[i] && prevBlocks[i]) {
@@ -192,8 +193,8 @@ class MempoolBlocks {
         mempoolBlocks[i].transactions.forEach(tx => {
           if (!prevIds[tx.txid]) {
             added.push(tx);
-          } else if (tx.rate !== prevIds[tx.txid].rate) {
-            changed.push({ txid: tx.txid, rate: tx.rate });
+          } else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) {
+            changed.push({ txid: tx.txid, rate: tx.rate, acc: tx.acc });
           }
         });
       }
@@ -206,14 +207,19 @@ class MempoolBlocks {
     return mempoolBlockDeltas;
   }
 
-  public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+  public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
-    this.resetUids();
-    for (const tx of Object.values(newMempool)) {
-      this.setUid(tx);
+    if (saveResults) {
+      this.resetUids();
     }
+    // set missing short ids
+    for (const tx of Object.values(newMempool)) {
+      this.setUid(tx, !saveResults);
+    }
+
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
 
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
@@ -222,7 +228,7 @@ class MempoolBlocks {
       if (entry.uid !== null && entry.uid !== undefined) {
         const stripped = {
           uid: entry.uid,
-          fee: entry.fee,
+          fee: entry.fee + (useAccelerations && (!accelerationPool || accelerations[entry.txid]?.pools?.includes(accelerationPool)) ? (accelerations[entry.txid]?.feeDelta || 0) : 0),
           weight: (entry.adjustedVsize * 4),
           sigops: entry.sigops,
           feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize,
@@ -262,7 +268,7 @@ class MempoolBlocks {
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults);
 
       logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
 
@@ -273,25 +279,29 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise<void> {
+  public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
     if (!this.txSelectionWorker) {
       // need to reset the worker
-      await this.$makeBlockTemplates(newMempool, saveResults);
+      await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations);
       return;
     }
 
     const start = Date.now();
 
-    for (const tx of Object.values(added)) {
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
+    const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added;
+
+    for (const tx of addedAndChanged) {
       this.setUid(tx, true);
     }
-    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
+    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
+
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
-    const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => {
+    const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => {
       return {
         uid: entry.uid || 0,
-        fee: entry.fee,
+        fee: entry.fee + (useAccelerations ? (accelerations[entry.txid]?.feeDelta || 0) : 0),
         weight: (entry.adjustedVsize * 4),
         sigops: entry.sigops,
         feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize,
@@ -318,7 +328,7 @@ class MempoolBlocks {
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults);
+      this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults);
       logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
     } catch (e) {
       logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
@@ -330,7 +340,7 @@ class MempoolBlocks {
     this.rustGbtGenerator = new GbtGenerator();
   }
 
-  private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
@@ -346,16 +356,25 @@ class MempoolBlocks {
       tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
     }
 
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
+    const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
+    const convertedAccelerations = acceleratedList.map(acc => {
+      return {
+        uid: this.getUid(newMempool[acc.txid]),
+        delta: acc.feeDelta,
+      };
+    });
+
     // run the block construction algorithm in a separate thread, and wait for a result
     const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
     try {
       const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
-        await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], this.nextUid),
+        await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
       );
       if (saveResults) {
         this.rustInitialized = true;
       }
-      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults);
       logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
       return processed;
     } catch (e) {
@@ -367,20 +386,20 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }): Promise<MempoolBlockWithTransactions[]> {
-    return this.$rustMakeBlockTemplates(newMempool, false);
+  public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+    return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool);
   }
 
-  public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[]): Promise<void> {
+  public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     // GBT optimization requires that uids never get too sparse
     // as a sanity check, we should also explicitly prevent uint32 uid overflow
     if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) {
       this.resetRustGbt();
     }
+
     if (!this.rustInitialized) {
       // need to reset the worker
-      await this.$rustMakeBlockTemplates(newMempool, true);
-      return;
+      return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool);
     }
 
     const start = Date.now();
@@ -394,12 +413,22 @@ class MempoolBlocks {
     }
     const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
 
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
+    const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
+    const convertedAccelerations = acceleratedList.map(acc => {
+      return {
+        uid: this.getUid(newMempool[acc.txid]),
+        delta: acc.feeDelta,
+      };
+    });
+
     // run the block construction algorithm in a separate thread, and wait for a result
     try {
       const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
         await this.rustGbtGenerator.update(
           added as RustThreadTransaction[],
           removedUids,
+          convertedAccelerations as RustThreadAcceleration[],
           this.nextUid,
         ),
       );
@@ -407,17 +436,19 @@ class MempoolBlocks {
       if (mempoolSize !== resultMempoolSize) {
         throw new Error('GBT returned wrong number of transactions, cache is probably out of sync');
       } else {
-        this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true);
+        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true);
+        this.removeUids(removedUids);
+        logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
+        return processed;
       }
-      this.removeUids(removedUids);
-      logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
     } catch (e) {
       logger.err('RUST updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
       this.resetRustGbt();
+      return this.mempoolBlocks;
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], saveResults): MempoolBlockWithTransactions[] {
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
     for (const [txid, rate] of rates) {
       if (txid in mempool) {
         mempool[txid].effectiveFeePerVsize = rate;
@@ -468,6 +499,8 @@ class MempoolBlocks {
       }
     }
 
+    const isAccelerated : { [txid: string]: boolean } = {};
+
     const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
     // update this thread's mempool with the results
     let mempoolTx: MempoolTransactionExtended;
@@ -496,6 +529,17 @@ class MempoolBlocks {
             mempoolTx.cpfpChecked = true;
           }
 
+          const acceleration = accelerations[txid];
+          if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+            mempoolTx.acceleration = true;
+            for (const ancestor of mempoolTx.ancestors || []) {
+              mempool[ancestor.txid].acceleration = true;
+              isAccelerated[ancestor.txid] = true;
+            }
+          } else {
+            delete mempoolTx.acceleration;
+          }
+
           // online calculation of stack-of-blocks fee stats
           if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) {
             feeStatsCalculator.processNext(mempoolTx);
@@ -532,7 +576,7 @@ class MempoolBlocks {
 
   private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
     if (!feeStats) {
-      feeStats = Common.calcEffectiveFeeStatistics(transactions);
+      feeStats = Common.calcEffectiveFeeStatistics(transactions.filter(tx => !tx.acceleration));
     }
     return {
       blockSize: totalSize,

backend/src/api/mempool.ts

@@ -9,6 +9,8 @@ import loadingIndicators from './loading-indicators';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
+import accelerationApi, { Acceleration } from './services/acceleration';
+import redisCache from './redis-cache';
 
 class Mempool {
   private inSync: boolean = false;
@@ -18,9 +20,11 @@ class Mempool {
   private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
-    deletedTransactions: MempoolTransactionExtended[]) => void) | undefined;
+    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined;
   private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
-    deletedTransactions: MempoolTransactionExtended[]) => Promise<void>) | undefined;
+    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>) | undefined;
+
+  private accelerations: { [txId: string]: Acceleration } = {};
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
@@ -65,12 +69,12 @@ class Mempool {
   }
 
   public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; },
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => void): void {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void {
     this.mempoolChangedCallback = fn;
   }
 
   public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise<void>): void {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>): void {
     this.$asyncMempoolChangedCallback = fn;
   }
 
@@ -85,6 +89,10 @@ class Mempool {
   public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
     this.mempoolCache = mempoolData;
     let count = 0;
+    const redisTimer = Date.now();
+    if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+      logger.debug(`Migrating ${Object.keys(this.mempoolCache).length} transactions from disk cache to Redis cache`);
+    }
     for (const txid of Object.keys(this.mempoolCache)) {
       if (!this.mempoolCache[txid].sigops || this.mempoolCache[txid].effectiveFeePerVsize == null) {
         this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]);
@@ -93,16 +101,61 @@ class Mempool {
         this.mempoolCache[txid].order = transactionUtils.txidToOrdering(txid);
       }
       count++;
+      if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+        await redisCache.$addTransaction(this.mempoolCache[txid]);
+      }
+    }
+    if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+      await redisCache.$flushTransactions();
+      logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
     }
     if (this.mempoolChangedCallback) {
-      this.mempoolChangedCallback(this.mempoolCache, [], []);
+      this.mempoolChangedCallback(this.mempoolCache, [], [], []);
     }
     if (this.$asyncMempoolChangedCallback) {
-      await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], []);
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []);
     }
     this.addToSpendMap(Object.values(this.mempoolCache));
   }
 
+  public async $reloadMempool(expectedCount: number): Promise<MempoolTransactionExtended[]> {
+    let count = 0;
+    let done = false;
+    let last_txid;
+    const newTransactions: MempoolTransactionExtended[] = [];
+    loadingIndicators.setProgress('mempool', count / expectedCount * 100);
+    while (!done) {
+      try {
+        const result = await bitcoinApi.$getAllMempoolTransactions(last_txid);
+        if (result) {
+          for (const tx of result) {
+            const extendedTransaction = transactionUtils.extendMempoolTransaction(tx);
+            if (!this.mempoolCache[extendedTransaction.txid]) {
+              newTransactions.push(extendedTransaction);
+              this.mempoolCache[extendedTransaction.txid] = extendedTransaction;
+            }
+            count++;
+          }
+          logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`);
+          if (result.length > 0) {
+            last_txid = result[result.length - 1].txid;
+          } else {
+            done = true;
+          }
+          if (Math.floor((count / expectedCount) * 100) < 100) {
+            loadingIndicators.setProgress('mempool', count / expectedCount * 100);
+          }
+        } else {
+          done = true;
+        }
+      } catch(err) {
+        logger.err('failed to fetch bulk mempool transactions from esplora');
+      }
+    }
+    logger.info(`Done inserting loaded mempool transactions into local cache`);
+    return newTransactions;
+  }
+
   public async $updateMemPoolInfo() {
     this.mempoolInfo = await this.$getMempoolInfo();
   }
@@ -132,7 +185,7 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool(transactions: string[]): Promise<void> {
+  public async $updateMempool(transactions: string[], pollRate: number): Promise<void> {
     logger.debug(`Updating mempool...`);
 
     // warn if this run stalls the main loop for more than 2 minutes
@@ -143,7 +196,7 @@ class Mempool {
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
     this.updateTimerProgress(timer, 'got raw mempool');
     const diff = transactions.length - currentMempoolSize;
-    const newTransactions: MempoolTransactionExtended[] = [];
+    let newTransactions: MempoolTransactionExtended[] = [];
 
     this.mempoolCacheDelta = Math.abs(diff);
 
@@ -162,12 +215,35 @@ class Mempool {
     };
 
     let intervalTimer = Date.now();
-    for (const txid of transactions) {
-      if (!this.mempoolCache[txid]) {
-        try {
-          const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false);
-          this.updateTimerProgress(timer, 'fetched new transaction');
-          this.mempoolCache[txid] = transaction;
+
+    let loaded = false;
+    if (config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize < transactions.length * 0.5 && transactions.length > 20_000) {
+      this.inSync = false;
+      logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`);
+      try {
+        newTransactions = await this.$reloadMempool(transactions.length);
+        if (config.REDIS.ENABLED) {
+          for (const tx of newTransactions) {
+            await redisCache.$addTransaction(tx);
+          }
+        }
+        loaded = true;
+      } catch (e) {
+        logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions');
+      }
+    }
+
+    if (!loaded) {
+      const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]);
+      const sliceLength = 10000;
+      for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) {
+        const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength);
+        const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false);
+        logger.debug(`fetched ${txs.length} transactions`);
+        this.updateTimerProgress(timer, 'fetched new transactions');
+
+        for (const transaction of txs) {
+          this.mempoolCache[transaction.txid] = transaction;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -177,26 +253,34 @@ class Mempool {
           }
           hasChange = true;
           newTransactions.push(transaction);
-        } catch (e: any) {
-          if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
-            this.missingTxCount++;
-          }
-          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
-        }
-      }
 
-      if (Date.now() - intervalTimer > 5_000) {
-        
-        if (this.inSync) {
-          // Break and restart mempool loop if we spend too much time processing
-          // new transactions that may lead to falling behind on block height
-          logger.debug('Breaking mempool loop because the 5s time limit exceeded.');
-          break;
-        } else {
-          const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100;
-          logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`);
-          loadingIndicators.setProgress('mempool', progress);
-          intervalTimer = Date.now()
+          if (config.REDIS.ENABLED) {
+            await redisCache.$addTransaction(transaction);
+          }
+        }
+
+        if (txs.length < slice.length) {
+          const missing = slice.length - txs.length;
+          if (config.MEMPOOL.BACKEND === 'esplora') {
+            this.missingTxCount += missing;
+          }
+          logger.debug(`Error finding ${missing} transactions in the mempool: `);
+        }
+
+        if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) {
+          if (this.inSync) {
+            // Break and restart mempool loop if we spend too much time processing
+            // new transactions that may lead to falling behind on block height
+            logger.debug('Breaking mempool loop because the 5s time limit exceeded.');
+            break;
+          } else {
+            const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100;
+            logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`);
+            if (Math.floor(progress) < 100) {
+              loadingIndicators.setProgress('mempool', progress);
+            }
+            intervalTimer = Date.now();
+          }
         }
       }
     }
@@ -219,7 +303,7 @@ class Mempool {
       logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`);
       setTimeout(() => {
         this.mempoolProtection = 2;
-        logger.warn('Mempool clear protection resumed.');
+        logger.warn('Mempool clear protection ended, normal operation resumed.');
       }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
     }
 
@@ -246,21 +330,33 @@ class Mempool {
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
+    const accelerationDelta = await this.$updateAccelerations();
+    if (accelerationDelta.length) {
+      hasChange = true;
+    }
+
+    this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
+
+    if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta);
+    }
+    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      this.updateTimerProgress(timer, 'running async mempool callback');
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta);
+      this.updateTimerProgress(timer, 'completed async mempool callback');
+    }
+
     if (!this.inSync && transactions.length === newMempoolSize) {
       this.inSync = true;
       logger.notice('The mempool is now in sync!');
       loadingIndicators.setProgress('mempool', 100);
     }
 
-    this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
-
-    if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
-    }
-    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      this.updateTimerProgress(timer, 'running async mempool callback');
-      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions);
-      this.updateTimerProgress(timer, 'completed async mempool callback');
+    // Update Redis cache
+    if (config.REDIS.ENABLED) {
+      await redisCache.$flushTransactions();
+      await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid));
+      await rbfCache.updateCache();
     }
 
     const end = new Date().getTime();
@@ -270,6 +366,70 @@ class Mempool {
     this.clearTimer(timer);
   }
 
+  public getAccelerations(): { [txid: string]: Acceleration } {
+    return this.accelerations;
+  }
+
+  public async $updateAccelerations(): Promise<string[]> {
+    if (!config.MEMPOOL_SERVICES.ACCELERATIONS) {
+      return [];
+    }
+
+    try {
+      const newAccelerations = await accelerationApi.$fetchAccelerations();
+
+      const changed: string[] = [];
+
+      const newAccelerationMap: { [txid: string]: Acceleration } = {};
+      for (const acceleration of newAccelerations) {
+        newAccelerationMap[acceleration.txid] = acceleration;
+        if (this.accelerations[acceleration.txid] == null) {
+          // new acceleration
+          changed.push(acceleration.txid);
+        } else {
+          if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) {
+            // feeDelta changed
+            changed.push(acceleration.txid);
+          } else if (this.accelerations[acceleration.txid].pools?.length) {
+            let poolsChanged = false;
+            const pools = new Set();
+            this.accelerations[acceleration.txid].pools.forEach(pool => {
+              pools.add(pool);
+            });
+            acceleration.pools.forEach(pool => {
+              if (!pools.has(pool)) {
+                poolsChanged = true;
+              } else {
+                pools.delete(pool);
+              }
+            });
+            if (pools.size > 0) {
+              poolsChanged = true;
+            }
+            if (poolsChanged) {
+              // pools changed
+              changed.push(acceleration.txid);
+            }
+          }
+        }
+      }
+
+      for (const oldTxid of Object.keys(this.accelerations)) {
+        if (!newAccelerationMap[oldTxid]) {
+          // removed
+          changed.push(oldTxid);
+        }
+      }
+
+      this.accelerations = newAccelerationMap;
+
+      return changed;
+    } catch (e: any) {
+      logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
   private startTimer() {
     const state: any = {
       start: Date.now(),

backend/src/api/mining/mining-routes.ts

@@ -12,6 +12,7 @@ import PricesRepository from '../../repositories/PricesRepository';
 class MiningRoutes {
   public initRoutes(app: Application) {
     app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools', this.$listPools)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', this.$getPools)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', this.$getPoolHistoricalHashrate)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', this.$getPoolBlocks)
@@ -41,6 +42,10 @@ class MiningRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      if (['testnet', 'signet', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
+        res.status(400).send('Prices are not available on testnets.');
+        return;
+      }
       if (req.query.timestamp) {
         res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
           parseInt(<string>req.query.timestamp ?? 0, 10)
@@ -88,6 +93,29 @@ class MiningRoutes {
     }
   }
 
+  private async $listPools(req: Request, res: Response): Promise<void> {
+    try {
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+
+      const pools = await mining.$listPools();
+      if (!pools) {
+        res.status(500).end();
+        return;
+      }
+
+      res.header('X-total-count', pools.length.toString());
+      if (pools.length === 0) {
+        res.status(204).send();
+      } else {
+        res.json(pools);
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getPools(req: Request, res: Response) {
     try {
       const stats = await mining.$getPoolsStats(req.params.interval);

backend/src/api/mining/mining.ts

@@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
 import config from '../../config';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import PricesRepository from '../../repositories/PricesRepository';
-import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
+import bitcoinApi from '../bitcoin/bitcoin-api-factory';
 import { IEsploraApi } from '../bitcoin/esplora-api.interface';
 import database from '../../database';
 
@@ -26,7 +26,7 @@ class Mining {
   /**
    * Get historical blocks health
    */
-   public async $getBlocksHealthHistory(interval: string | null = null): Promise<any> {
+  public async $getBlocksHealthHistory(interval: string | null = null): Promise<any> {
     return await BlocksAuditsRepository.$getBlocksHealthHistory(
       this.getTimeRange(interval),
       Common.getSqlInterval(interval)
@@ -56,7 +56,7 @@ class Mining {
   /**
    * Get historical block fee rates percentiles
    */
-   public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
+  public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
     return await BlocksRepository.$getHistoricalBlockFeeRates(
       this.getTimeRange(interval),
       Common.getSqlInterval(interval)
@@ -66,7 +66,7 @@ class Mining {
   /**
    * Get historical block sizes
    */
-   public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
+  public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
     return await BlocksRepository.$getHistoricalBlockSizes(
       this.getTimeRange(interval),
       Common.getSqlInterval(interval)
@@ -76,7 +76,7 @@ class Mining {
   /**
    * Get historical block weights
    */
-   public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
+  public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
     return await BlocksRepository.$getHistoricalBlockWeights(
       this.getTimeRange(interval),
       Common.getSqlInterval(interval)
@@ -107,6 +107,7 @@ class Mining {
         slug: poolInfo.slug,
         avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
         avgFeeDelta: poolInfo.avgFeeDelta,
+        poolUniqueId: poolInfo.poolUniqueId
       };
       poolsStats.push(poolStat);
     });
@@ -201,7 +202,7 @@ class Mining {
     try {
       const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
-      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
       const genesisTimestamp = genesisBlock.timestamp * 1000;
 
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
@@ -312,7 +313,7 @@ class Mining {
     const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
     try {
-      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
       const genesisTimestamp = genesisBlock.timestamp * 1000;
       const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       const lastMidnight = this.getDateMidnight(new Date());
@@ -421,8 +422,9 @@ class Mining {
     }
 
     const blocks: any = await BlocksRepository.$getBlocksDifficulty();
-    const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+    const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
     let currentDifficulty = genesisBlock.difficulty;
+    let currentBits = genesisBlock.bits;
     let totalIndexed = 0;
 
     if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
@@ -436,6 +438,7 @@ class Mining {
 
     const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock();
     if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) {
+      currentBits = oldestConsecutiveBlock.bits;
       currentDifficulty = oldestConsecutiveBlock.difficulty;
     }
 
@@ -443,10 +446,11 @@ class Mining {
     let timer = new Date().getTime() / 1000;
 
     for (const block of blocks) {
-      if (block.difficulty !== currentDifficulty) {
+      if (block.bits !== currentBits) {
         if (indexedHeights[block.height] === true) { // Already indexed
           if (block.height >= oldestConsecutiveBlock.height) {
             currentDifficulty = block.difficulty;
+            currentBits = block.bits;
           }
           continue;          
         }
@@ -464,6 +468,7 @@ class Mining {
         totalIndexed++;
         if (block.height >= oldestConsecutiveBlock.height) {
           currentDifficulty = block.difficulty;
+          currentBits = block.bits;
         }
     }
 
@@ -590,6 +595,20 @@ class Mining {
     }
   }
 
+  /**
+   * List existing mining pools
+   */
+  public async $listPools(): Promise<{name: string, slug: string, unique_id: number}[] | null> {
+    const [rows] = await database.query(`
+      SELECT
+        name,
+        slug,
+        unique_id
+      FROM pools`
+    );
+    return rows as {name: string, slug: string, unique_id: number}[];
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/pools-parser.ts

@@ -131,7 +131,7 @@ class PoolsParser {
     let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
     if (config.MEMPOOL.NETWORK === 'testnet') {
       firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
-    } else if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'regtest') {
+    } else if (config.MEMPOOL.NETWORK === 'signet') {
       firstKnownBlockPool = 0;
     }
 
@@ -159,7 +159,7 @@ class PoolsParser {
     let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
     if (config.MEMPOOL.NETWORK === 'testnet') {
       firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
-    } else if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'regtest') {
+    } else if (config.MEMPOOL.NETWORK === 'signet') {
       firstKnownBlockPool = 0;
     }
 

backend/src/api/prices/prices.routes.ts

@@ -0,0 +1,19 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import pricesUpdater from '../../tasks/price-updater';
+
+class PricesRoutes {
+  public initRoutes(app: Application): void {
+    app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this));
+  }
+
+  private $getCurrentPrices(req: Request, res: Response): void {
+    res.header('Pragma', 'public');
+    res.header('Cache-control', 'public');
+    res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString());
+
+    res.json(pricesUpdater.getLatestPrices());
+  }
+}
+
+export default new PricesRoutes();

backend/src/api/rbf-cache.ts

@@ -1,15 +1,17 @@
+import config from "../config";
 import logger from "../logger";
 import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces";
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import { Common } from "./common";
+import redisCache from "./redis-cache";
 
-interface RbfTransaction extends TransactionStripped {
+export interface RbfTransaction extends TransactionStripped {
   rbf?: boolean;
   mined?: boolean;
   fullRbf?: boolean;
 }
 
-interface RbfTree {
+export interface RbfTree {
   tx: RbfTransaction;
   time: number;
   interval?: number;
@@ -28,6 +30,19 @@ export interface ReplacementInfo {
   newVsize: number;
 }
 
+enum CacheOp {
+  Remove = 0,
+  Add = 1,
+  Change = 2,
+}
+
+interface CacheEvent {
+  op: CacheOp;
+  type: 'tx' | 'tree' | 'exp';
+  txid: string,
+  value?: any,
+}
+
 class RbfCache {
   private replacedBy: Map<string, string> = new Map();
   private replaces: Map<string, string[]> = new Map();
@@ -36,11 +51,43 @@ class RbfCache {
   private treeMap: Map<string, string> = new Map(); // map of txids to sequence ids
   private txs: Map<string, MempoolTransactionExtended> = new Map();
   private expiring: Map<string, number> = new Map();
+  private cacheQueue: CacheEvent[] = [];
 
   constructor() {
     setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
   }
 
+  private addTx(txid: string, tx: MempoolTransactionExtended): void {
+    this.txs.set(txid, tx);
+    this.cacheQueue.push({ op: CacheOp.Add, type: 'tx', txid });
+  }
+
+  private addTree(txid: string, tree: RbfTree): void {
+    this.rbfTrees.set(txid, tree);
+    this.dirtyTrees.add(txid);
+    this.cacheQueue.push({ op: CacheOp.Add, type: 'tree', txid });
+  }
+
+  private addExpiration(txid: string, expiry: number): void {
+    this.expiring.set(txid, expiry);
+    this.cacheQueue.push({ op: CacheOp.Add, type: 'exp', txid, value: expiry });
+  }
+
+  private removeTx(txid: string): void {
+    this.txs.delete(txid);
+    this.cacheQueue.push({ op: CacheOp.Remove, type: 'tx', txid });
+  }
+
+  private removeTree(txid: string): void {
+    this.rbfTrees.delete(txid);
+    this.cacheQueue.push({ op: CacheOp.Remove, type: 'tree', txid });
+  }
+
+  private removeExpiration(txid: string): void {
+    this.expiring.delete(txid);
+    this.cacheQueue.push({ op: CacheOp.Remove, type: 'exp', txid });
+  }
+
   public add(replaced: MempoolTransactionExtended[], newTxExtended: MempoolTransactionExtended): void {
     if (!newTxExtended || !replaced?.length || this.txs.has(newTxExtended.txid)) {
       return;
@@ -49,7 +96,7 @@ class RbfCache {
     const newTx = Common.stripTransaction(newTxExtended) as RbfTransaction;
     const newTime = newTxExtended.firstSeen || (Date.now() / 1000);
     newTx.rbf = newTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
-    this.txs.set(newTx.txid, newTxExtended);
+    this.addTx(newTx.txid, newTxExtended);
 
     // maintain rbf trees
     let txFullRbf = false;
@@ -66,7 +113,7 @@ class RbfCache {
         const treeId = this.treeMap.get(replacedTx.txid);
         if (treeId) {
           const tree = this.rbfTrees.get(treeId);
-          this.rbfTrees.delete(treeId);
+          this.removeTree(treeId);
           if (tree) {
             tree.interval = newTime - tree?.time;
             replacedTrees.push(tree);
@@ -83,7 +130,7 @@ class RbfCache {
           replaces: [],
         });
         treeFullRbf = treeFullRbf || !replacedTx.rbf;
-        this.txs.set(replacedTx.txid, replacedTxExtended);
+        this.addTx(replacedTx.txid, replacedTxExtended);
       }
     }
     newTx.fullRbf = txFullRbf;
@@ -94,10 +141,27 @@ class RbfCache {
       fullRbf: treeFullRbf,
       replaces: replacedTrees
     };
-    this.rbfTrees.set(treeId, newTree);
+    this.addTree(treeId, newTree);
     this.updateTreeMap(treeId, newTree);
     this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid));
-    this.dirtyTrees.add(treeId);
+  }
+
+  public has(txId: string): boolean {
+    return this.txs.has(txId);
+  }
+
+  public anyInSameTree(txId: string, predicate: (tx: RbfTransaction) => boolean): boolean {
+    const tree = this.getRbfTree(txId);
+    if (!tree) {
+      return false;
+    }
+    const txs = this.getTransactionsInTree(tree);
+    for (const tx of txs) {
+      if (predicate(tx)) {
+        return true;
+      }
+    }
+    return false;
   }
 
   public getReplacedBy(txId: string): string | undefined {
@@ -173,6 +237,7 @@ class RbfCache {
         this.setTreeMined(tree, txid);
         tree.mined = true;
         this.dirtyTrees.add(treeId);
+        this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId });
       }
     }
     this.evict(txid);
@@ -181,7 +246,8 @@ class RbfCache {
   // flag a transaction as removed from the mempool
   public evict(txid: string, fast: boolean = false): void {
     if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
-      this.expiring.set(txid, fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400)); // 24 hours
+      const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
+      this.addExpiration(txid, expiryTime);
     }
   }
 
@@ -202,11 +268,11 @@ class RbfCache {
     const now = Date.now();
     for (const txid of this.expiring.keys()) {
       if ((this.expiring.get(txid) || 0) < now) {
-        this.expiring.delete(txid);
+        this.removeExpiration(txid);
         this.remove(txid);
       }
     }
-    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.expiring.size} due to expire`);
+    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire`);
   }
 
   // remove a transaction & all previous versions from the cache
@@ -216,14 +282,14 @@ class RbfCache {
       const replaces = this.replaces.get(txid);
       this.replaces.delete(txid);
       this.treeMap.delete(txid);
-      this.txs.delete(txid);
-      this.expiring.delete(txid);
+      this.removeTx(txid);
+      this.removeExpiration(txid);
       for (const tx of (replaces || [])) {
         // recursively remove prior versions from the cache
         this.replacedBy.delete(tx);
         // if this is the id of a tree, remove that too
         if (this.treeMap.get(tx) === tx) {
-          this.rbfTrees.delete(tx);
+          this.removeTree(tx);
         }
         this.remove(tx);
       }
@@ -255,6 +321,33 @@ class RbfCache {
     }
   }
 
+  public async updateCache(): Promise<void> {
+    if (!config.REDIS.ENABLED) {
+      return;
+    }
+    // Update the Redis cache by replaying queued events
+    for (const e of this.cacheQueue) {
+      if (e.op === CacheOp.Add || e.op === CacheOp.Change) {
+        let value = e.value;
+          switch(e.type) {
+            case 'tx': {
+              value = this.txs.get(e.txid);
+            } break;
+            case 'tree': {
+              const tree = this.rbfTrees.get(e.txid);
+              value = tree ? this.exportTree(tree) : null;
+            } break;
+          }
+          if (value != null) {
+            await redisCache.$setRbfEntry(e.type, e.txid, value);
+          }
+      } else if (e.op === CacheOp.Remove) {
+        await redisCache.$removeRbfEntry(e.type, e.txid);
+      }
+    }
+    this.cacheQueue = [];
+  }
+
   public dump(): any {
     const trees = Array.from(this.rbfTrees.values()).map((tree: RbfTree) => { return this.exportTree(tree); });
 
@@ -267,14 +360,14 @@ class RbfCache {
 
   public async load({ txs, trees, expiring }): Promise<void> {
     txs.forEach(txEntry => {
-      this.txs.set(txEntry[0], txEntry[1]);
+      this.txs.set(txEntry.key, txEntry.value);
     });
     for (const deflatedTree of trees) {
       await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
     }
     expiring.forEach(expiringEntry => {
-      if (this.txs.has(expiringEntry[0])) {
-        this.expiring.set(expiringEntry[0], new Date(expiringEntry[1]).getTime());
+      if (this.txs.has(expiringEntry.key)) {
+        this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
       }
     });
     this.cleanup();
@@ -360,8 +453,7 @@ class RbfCache {
     };
     this.treeMap.set(txid, root);
     if (root === txid) {
-      this.rbfTrees.set(root, tree);
-      this.dirtyTrees.add(root);
+      this.addTree(root, tree);
     }
     return tree;
   }

backend/src/api/redis-cache.ts

@@ -0,0 +1,276 @@
+import { createClient } from 'redis';
+import memPool from './mempool';
+import blocks from './blocks';
+import logger from '../logger';
+import config from '../config';
+import { BlockExtended, BlockSummary, MempoolTransactionExtended } from '../mempool.interfaces';
+import rbfCache from './rbf-cache';
+import transactionUtils from './transaction-utils';
+
+enum NetworkDB {
+  mainnet = 0,
+  testnet,
+  signet,
+  liquid,
+  liquidtestnet,
+}
+
+class RedisCache {
+  private client;
+  private connected = false;
+  private schemaVersion = 1;
+
+  private cacheQueue: MempoolTransactionExtended[] = [];
+  private txFlushLimit: number = 10000;
+
+  constructor() {
+    if (config.REDIS.ENABLED) {
+      const redisConfig = {
+        socket: {
+          path: config.REDIS.UNIX_SOCKET_PATH
+        },
+        database: NetworkDB[config.MEMPOOL.NETWORK],
+      };
+      this.client = createClient(redisConfig);
+      this.client.on('error', (e) => {
+        logger.err(`Error in Redis client: ${e instanceof Error ? e.message : e}`);
+      });
+      this.$ensureConnected();
+    }
+  }
+
+  private async $ensureConnected(): Promise<void> {
+    if (!this.connected && config.REDIS.ENABLED) {
+      return this.client.connect().then(async () => {
+        this.connected = true;
+        logger.info(`Redis client connected`);
+        const version = await this.client.get('schema_version');
+        if (version !== this.schemaVersion) {
+          // schema changed
+          // perform migrations or flush DB if necessary
+          logger.info(`Redis schema version changed from ${version} to ${this.schemaVersion}`);
+          await this.client.set('schema_version', this.schemaVersion);
+        }
+      });
+    }
+  }
+
+  async $updateBlocks(blocks: BlockExtended[]) {
+    try {
+      await this.$ensureConnected();
+      await this.client.set('blocks', JSON.stringify(blocks));
+      logger.debug(`Saved latest blocks to Redis cache`);
+    } catch (e) {
+      logger.warn(`Failed to update blocks in Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $updateBlockSummaries(summaries: BlockSummary[]) {
+    try {
+      await this.$ensureConnected();
+      await this.client.set('block-summaries', JSON.stringify(summaries));
+      logger.debug(`Saved latest block summaries to Redis cache`);
+    } catch (e) {
+      logger.warn(`Failed to update block summaries in Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $addTransaction(tx: MempoolTransactionExtended) {
+    this.cacheQueue.push(tx);
+    if (this.cacheQueue.length >= this.txFlushLimit) {
+      await this.$flushTransactions();
+    }
+  }
+
+  async $flushTransactions() {
+    const success = await this.$addTransactions(this.cacheQueue);
+    if (success) {
+      logger.debug(`Saved ${this.cacheQueue.length} transactions to Redis cache`);
+      this.cacheQueue = [];
+    } else {
+      logger.err(`Failed to save ${this.cacheQueue.length} transactions to Redis cache`);
+    }
+  }
+
+  private async $addTransactions(newTransactions: MempoolTransactionExtended[]): Promise<boolean> {
+    if (!newTransactions.length) {
+      return true;
+    }
+    try {
+      await this.$ensureConnected();
+      const msetData = newTransactions.map(tx => {
+        const minified: any = { ...tx };
+        delete minified.hex;
+        for (const vin of minified.vin) {
+          delete vin.inner_redeemscript_asm;
+          delete vin.inner_witnessscript_asm;
+          delete vin.scriptsig_asm;
+        }
+        for (const vout of minified.vout) {
+          delete vout.scriptpubkey_asm;
+        }
+        return [`mempool:tx:${tx.txid}`, JSON.stringify(minified)];
+      });
+      await this.client.MSET(msetData);
+      return true;
+    } catch (e) {
+      logger.warn(`Failed to add ${newTransactions.length} transactions to Redis cache: ${e instanceof Error ? e.message : e}`);
+      return false;
+    }
+  }
+
+  async $removeTransactions(transactions: string[]) {
+    try {
+      await this.$ensureConnected();
+      for (let i = 0; i < Math.ceil(transactions.length / 10000); i++) {
+        const slice = transactions.slice(i * 10000, (i + 1) * 10000);
+        await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`));
+        logger.debug(`Deleted ${slice.length} transactions from the Redis cache`);
+      }
+    } catch (e) {
+      logger.warn(`Failed to remove ${transactions.length} transactions from Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $setRbfEntry(type: string, txid: string, value: any): Promise<void> {
+    try {
+      await this.$ensureConnected();
+      await this.client.set(`rbf:${type}:${txid}`, JSON.stringify(value));
+    } catch (e) {
+      logger.warn(`Failed to set RBF ${type} in Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $removeRbfEntry(type: string, txid: string): Promise<void> {
+    try {
+      await this.$ensureConnected();
+      await this.client.unlink(`rbf:${type}:${txid}`);
+    } catch (e) {
+      logger.warn(`Failed to remove RBF ${type} from Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $getBlocks(): Promise<BlockExtended[]> {
+    try {
+      await this.$ensureConnected();
+      const json = await this.client.get('blocks');
+      return JSON.parse(json);
+    } catch (e) {
+      logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
+      return [];
+    }
+  }
+
+  async $getBlockSummaries(): Promise<BlockSummary[]> {
+    try {
+      await this.$ensureConnected();
+      const json = await this.client.get('block-summaries');
+      return JSON.parse(json);
+    } catch (e) {
+      logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
+      return [];
+    }
+  }
+
+  async $getMempool(): Promise<{ [txid: string]: MempoolTransactionExtended }> {
+    const start = Date.now();
+    const mempool = {};
+    try {
+      await this.$ensureConnected();
+      const mempoolList = await this.scanKeys<MempoolTransactionExtended>('mempool:tx:*');
+      for (const tx of mempoolList) {
+        mempool[tx.key] = tx.value;
+      }
+      logger.info(`Loaded mempool from Redis cache in ${Date.now() - start} ms`);
+      return mempool || {};
+    } catch (e) {
+      logger.warn(`Failed to retrieve mempool from Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+    return {};
+  }
+
+  async $getRbfEntries(type: string): Promise<any[]> {
+    try {
+      await this.$ensureConnected();
+      const rbfEntries = await this.scanKeys<MempoolTransactionExtended[]>(`rbf:${type}:*`);
+      return rbfEntries;
+    } catch (e) {
+      logger.warn(`Failed to retrieve Rbf ${type}s from Redis cache: ${e instanceof Error ? e.message : e}`);
+      return [];
+    }
+  }
+
+  async $loadCache() {
+    logger.info('Restoring mempool and blocks data from Redis cache');
+    // Load block data
+    const loadedBlocks = await this.$getBlocks();
+    const loadedBlockSummaries = await this.$getBlockSummaries();
+    // Load mempool
+    const loadedMempool = await this.$getMempool();
+    this.inflateLoadedTxs(loadedMempool);
+    // Load rbf data
+    const rbfTxs = await this.$getRbfEntries('tx');
+    const rbfTrees = await this.$getRbfEntries('tree');
+    const rbfExpirations = await this.$getRbfEntries('exp');
+
+    // Set loaded data
+    blocks.setBlocks(loadedBlocks || []);
+    blocks.setBlockSummaries(loadedBlockSummaries || []);
+    await memPool.$setMempool(loadedMempool);
+    await rbfCache.load({
+      txs: rbfTxs,
+      trees: rbfTrees.map(loadedTree => loadedTree.value),
+      expiring: rbfExpirations,
+    });
+  }
+
+  private inflateLoadedTxs(mempool: { [txid: string]: MempoolTransactionExtended }) {
+    for (const tx of Object.values(mempool)) {
+      for (const vin of tx.vin) {
+        if (vin.scriptsig) {
+          vin.scriptsig_asm = transactionUtils.convertScriptSigAsm(vin.scriptsig);
+          transactionUtils.addInnerScriptsToVin(vin);
+        }
+      }
+      for (const vout of tx.vout) {
+        if (vout.scriptpubkey) {
+          vout.scriptpubkey_asm = transactionUtils.convertScriptSigAsm(vout.scriptpubkey);
+        }
+      }
+    }
+  }
+
+  private async scanKeys<T>(pattern): Promise<{ key: string, value: T }[]> {
+    logger.info(`loading Redis entries for ${pattern}`);
+    let keys: string[] = [];
+    const result: { key: string, value: T }[] = [];
+    const patternLength = pattern.length - 1;
+    let count = 0;
+    const processValues = async (keys): Promise<void> => {
+      const values = await this.client.MGET(keys);
+      for (let i = 0; i < values.length; i++) {
+        if (values[i]) {
+          result.push({ key: keys[i].slice(patternLength), value: JSON.parse(values[i]) });
+          count++;
+        }
+      }
+      logger.info(`loaded ${count} entries from Redis cache`);
+    };
+    for await (const key of this.client.scanIterator({
+      MATCH: pattern,
+      COUNT: 100
+    })) {
+      keys.push(key);
+      if (keys.length >= 10000) {
+        await processValues(keys);
+        keys = [];
+      }
+    }
+    if (keys.length) {
+      await processValues(keys);
+    }
+    return result;
+  }
+}
+
+export default new RedisCache();

backend/src/api/services/acceleration.ts

@@ -0,0 +1,30 @@
+import { query } from '../../utils/axios-query';
+import config from '../../config';
+import { BlockExtended, PoolTag } from '../../mempool.interfaces';
+
+export interface Acceleration {
+  txid: string,
+  feeDelta: number,
+  pools: number[],
+}
+
+class AccelerationApi {
+  public async $fetchAccelerations(): Promise<Acceleration[]> {
+    if (config.MEMPOOL_SERVICES.ACCELERATIONS) {
+      const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations`);
+      return (response as Acceleration[]) || [];
+    } else {
+      return [];
+    }
+  }
+
+  public isAcceleratedBlock(block: BlockExtended, accelerations: Acceleration[]): boolean {
+    let anyAccelerated = false;
+    for (let i = 0; i < accelerations.length && !anyAccelerated; i++) {
+      anyAccelerated = anyAccelerated || accelerations[i].pools?.includes(block.extras.pool.id);
+    }
+    return anyAccelerated;
+  }
+}
+
+export default new AccelerationApi();

backend/src/api/transaction-utils.ts

@@ -3,6 +3,9 @@ import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import { Common } from './common';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import * as bitcoinjs from 'bitcoinjs-lib';
+import logger from '../logger';
+import config from '../config';
+import pLimit from '../utils/p-limit';
 
 class TransactionUtils {
   constructor() { }
@@ -22,6 +25,23 @@ class TransactionUtils {
     };
   }
 
+  // Wrapper for $getTransactionExtended with an automatic retry direct to Core if the first API request fails.
+  // Propagates any error from the retry request.
+  public async $getTransactionExtendedRetry(txid: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise<TransactionExtended> {
+    try {
+      const result = await this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, addMempoolData);
+      if (result) {
+        return result;
+      } else {
+        logger.err(`Cannot fetch tx ${txid}. Reason: backend returned null data`);
+      }
+    } catch (e) {
+      logger.err(`Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+    // retry direct from Core if first request failed
+    return this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, true, addMempoolData);
+  }
+
   /**
    * @param txId
    * @param addPrevouts
@@ -31,7 +51,7 @@ class TransactionUtils {
   public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise<TransactionExtended> {
     let transaction: IEsploraApi.Transaction;
     if (forceCore === true) {
-      transaction  = await bitcoinCoreApi.$getRawTransaction(txId, true);
+      transaction  = await bitcoinCoreApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
     } else {
       transaction  = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
     }
@@ -53,7 +73,29 @@ class TransactionUtils {
     return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended;
   }
 
-  private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
+  public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<MempoolTransactionExtended[]> {
+    if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') {
+      const limiter = pLimit(32); // Run 32 requests at a time
+      const results = await Promise.allSettled(txids.map(
+        txid => limiter(() => this.$getMempoolTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore))
+      ));
+      return results.filter(reply => reply.status === 'fulfilled')
+        .map(r => (r as PromiseFulfilledResult<MempoolTransactionExtended>).value);
+    } else {
+      const transactions = await bitcoinApi.$getMempoolTransactions(txids);
+      return transactions.map(transaction => {
+        if (Common.isLiquid()) {
+          if (!isFinite(Number(transaction.fee))) {
+            transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0);
+          }
+        }
+
+        return this.extendMempoolTransaction(transaction);
+      });
+    }
+  }
+
+  public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
     // @ts-ignore
     if (transaction.vsize) {
       // @ts-ignore
@@ -170,6 +212,122 @@ class TransactionUtils {
       16
     );
   }
+
+  public addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
+    if (!vin.prevout) {
+      return;
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'p2sh') {
+      const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
+      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
+      if (vin.witness && vin.witness.length > 2) {
+        const witnessScript = vin.witness[vin.witness.length - 1];
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      }
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
+      const witnessScript = vin.witness[vin.witness.length - 1];
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) {
+      const witnessScript = this.witnessToP2TRScript(vin.witness);
+      if (witnessScript !== null) {
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      }
+    }
+  }
+
+  public convertScriptSigAsm(hex: string): string {
+    const buf = Buffer.from(hex, 'hex');
+
+    const b: string[] = [];
+
+    let i = 0;
+    while (i < buf.length) {
+      const op = buf[i];
+      if (op >= 0x01 && op <= 0x4e) {
+        i++;
+        let push: number;
+        if (op === 0x4c) {
+          push = buf.readUInt8(i);
+          b.push('OP_PUSHDATA1');
+          i += 1;
+        } else if (op === 0x4d) {
+          push = buf.readUInt16LE(i);
+          b.push('OP_PUSHDATA2');
+          i += 2;
+        } else if (op === 0x4e) {
+          push = buf.readUInt32LE(i);
+          b.push('OP_PUSHDATA4');
+          i += 4;
+        } else {
+          push = op;
+          b.push('OP_PUSHBYTES_' + push);
+        }
+
+        const data = buf.slice(i, i + push);
+        if (data.length !== push) {
+          break;
+        }
+
+        b.push(data.toString('hex'));
+        i += data.length;
+      } else {
+        if (op === 0x00) {
+          b.push('OP_0');
+        } else if (op === 0x4f) {
+          b.push('OP_PUSHNUM_NEG1');
+        } else if (op === 0xb1) {
+          b.push('OP_CLTV');
+        } else if (op === 0xb2) {
+          b.push('OP_CSV');
+        } else if (op === 0xba) {
+          b.push('OP_CHECKSIGADD');
+        } else {
+          const opcode = bitcoinjs.script.toASM([ op ]);
+          if (opcode && op < 0xfd) {
+            if (/^OP_(\d+)$/.test(opcode)) {
+              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
+            } else {
+              b.push(opcode);
+            }
+          } else {
+            b.push('OP_RETURN_' + op);
+          }
+        }
+        i += 1;
+      }
+    }
+
+    return b.join(' ');
+  }
+
+  /**
+   * This function must only be called when we know the witness we are parsing
+   * is a taproot witness.
+   * @param witness An array of hex strings that represents the witness stack of
+   *                the input.
+   * @returns null if the witness is not a script spend, and the hex string of
+   *          the script item if it is a script spend.
+   */
+  public witnessToP2TRScript(witness: string[]): string | null {
+    if (witness.length < 2) return null;
+    // Note: see BIP341 for parsing details of witness stack
+
+    // If there are at least two witness elements, and the first byte of the
+    // last element is 0x50, this last element is called annex a and
+    // is removed from the witness stack.
+    const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50';
+    // If there are at least two witness elements left, script path spending is used.
+    // Call the second-to-last stack element s, the script.
+    // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
+    if (hasAnnex && witness.length < 3) return null;
+    const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
+    return witness[positionOfScript];
+  }
 }
 
 export default new TransactionUtils();

backend/src/api/websocket-handler.ts

@@ -21,6 +21,8 @@ import Audit from './audit';
 import { deepClone } from '../utils/clone';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
+import accelerationApi from './services/acceleration';
+import mempool from './mempool';
 
 // valid 'want' subscriptions
 const wantable = [
@@ -172,9 +174,15 @@ class WebsocketHandler {
               }
               const tx = memPool.getMempool()[trackTxid];
               if (tx && tx.position) {
+                const position: { block: number, vsize: number, accelerated?: boolean } = {
+                  ...tx.position
+                };
+                if (tx.acceleration) {
+                  position.accelerated = tx.acceleration;
+                }
                 response['txPosition'] = JSON.stringify({
                   txid: trackTxid,
-                  position: tx.position,
+                  position
                 });
               }
             } else {
@@ -183,13 +191,19 @@ class WebsocketHandler {
           }
 
           if (parsedMessage && parsedMessage['track-address']) {
-            if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100})$/
+            if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/
               .test(parsedMessage['track-address'])) {
               let matchedAddress = parsedMessage['track-address'];
               if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(parsedMessage['track-address'])) {
                 matchedAddress = matchedAddress.toLowerCase();
               }
-              client['track-address'] = matchedAddress;
+              if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) {
+                client['track-address'] = '41' + matchedAddress + 'ac';
+              } else if (/^(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) {
+                client['track-address'] = '21' + matchedAddress + 'ac';
+              } else {
+                client['track-address'] = matchedAddress;
+              }
             } else {
               client['track-address'] = null;
             }
@@ -380,7 +394,7 @@ class WebsocketHandler {
   }
 
   async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise<void> {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
@@ -389,9 +403,9 @@ class WebsocketHandler {
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
       if (config.MEMPOOL.RUST_GBT) {
-        await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions);
+        await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS);
       } else {
-        await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true);
+        await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
       }
     } else {
       mempoolBlocks.updateMempoolBlocks(newMempool, true);
@@ -470,6 +484,9 @@ class WebsocketHandler {
       }
     }
 
+    // pre-compute address transactions
+    const addressCache = this.makeAddressCache(newTransactions);
+
     this.wss.clients.forEach(async (client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
@@ -509,40 +526,13 @@ class WebsocketHandler {
       }
 
       if (client['track-address']) {
-        const foundTransactions: TransactionExtended[] = [];
+        const foundTransactions = Array.from(addressCache[client['track-address']]?.values() || []);
+        // txs may be missing prevouts in non-esplora backends
+        // so fetch the full transactions now
+        const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(foundTransactions) : foundTransactions;
 
-        for (const tx of newTransactions) {
-          const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address']);
-          if (someVin) {
-            if (config.MEMPOOL.BACKEND !== 'esplora') {
-              try {
-                const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true);
-                foundTransactions.push(fullTx);
-              } catch (e) {
-                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
-              }
-            } else {
-              foundTransactions.push(tx);
-            }
-            return;
-          }
-          const someVout = tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address']);
-          if (someVout) {
-            if (config.MEMPOOL.BACKEND !== 'esplora') {
-              try {
-                const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true);
-                foundTransactions.push(fullTx);
-              } catch (e) {
-                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
-              }
-            } else {
-              foundTransactions.push(tx);
-            }
-          }
-        }
-
-        if (foundTransactions.length) {
-          response['address-transactions'] = JSON.stringify(foundTransactions);
+        if (fullTransactions.length) {
+          response['address-transactions'] = JSON.stringify(fullTransactions);
         }
       }
 
@@ -550,7 +540,6 @@ class WebsocketHandler {
         const foundTransactions: TransactionExtended[] = [];
 
         newTransactions.forEach((tx) => {
-
           if (client['track-asset'] === Common.nativeAssetId) {
             if (tx.vin.some((vin) => !!vin.is_pegin)) {
               foundTransactions.push(tx);
@@ -599,12 +588,15 @@ class WebsocketHandler {
         if (mempoolTx && mempoolTx.position) {
           response['txPosition'] = JSON.stringify({
             txid: trackTxid,
-            position: mempoolTx.position,
+            position: {
+              ...mempoolTx.position,
+              accelerated: mempoolTx.acceleration || undefined,
+            }
           });
         }
       }
 
-      if (client['track-mempool-block'] >= 0) {
+      if (client['track-mempool-block'] >= 0 && memPool.isInSync()) {
         const index = client['track-mempool-block'];
         if (mBlockDeltas[index]) {
           response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, {
@@ -644,9 +636,10 @@ class WebsocketHandler {
     memPool.handleMinedRbfTransactions(rbfTransactions);
     memPool.removeFromSpendMap(transactions);
 
-    if (config.MEMPOOL.AUDIT) {
+    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
       let projectedBlocks;
       let auditMempool = _memPool;
+      const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
       // template calculation functions have mempool side effects, so calculate audits using
       // a cloned copy of the mempool if we're running a different algorithm for mempool updates
       const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
@@ -654,19 +647,27 @@ class WebsocketHandler {
         auditMempool = deepClone(_memPool);
         if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
           if (config.MEMPOOL.RUST_GBT) {
-            projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool);
+            projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id);
           } else {
-            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false);
+            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
           }
         } else {
           projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
         }
       } else {
-        projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+        if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
+          if (config.MEMPOOL.RUST_GBT) {
+            projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id);
+          } else {
+            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
+          }
+        } else {
+          projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+        }
       }
 
-      if (Common.indexingEnabled() && memPool.isInSync()) {
-        const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+      if (Common.indexingEnabled()) {
+        const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
         const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -695,6 +696,7 @@ class WebsocketHandler {
           freshTxs: fresh,
           sigopTxs: sigop,
           fullrbfTxs: fullrbf,
+          acceleratedTxs: accelerated,
           matchRate: matchRate,
           expectedFees: totalFees,
           expectedWeight: totalWeight,
@@ -722,9 +724,9 @@ class WebsocketHandler {
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
       if (config.MEMPOOL.RUST_GBT) {
-        await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions);
+        await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true);
       } else {
-        await mempoolBlocks.$makeBlockTemplates(_memPool, true);
+        await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
       }
     } else {
       mempoolBlocks.updateMempoolBlocks(_memPool, true);
@@ -736,6 +738,9 @@ class WebsocketHandler {
     const fees = feeApi.getRecommendedFee();
     const mempoolInfo = memPool.getMempoolInfo();
 
+    // pre-compute address transactions
+    const addressCache = this.makeAddressCache(transactions);
+
     // update init data
     this.updateSocketDataFields({
       'mempoolInfo': mempoolInfo,
@@ -788,24 +793,17 @@ class WebsocketHandler {
           if (mempoolTx && mempoolTx.position) {
             response['txPosition'] = JSON.stringify({
               txid: trackTxid,
-              position: mempoolTx.position,
+              position: {
+                ...mempoolTx.position,
+                accelerated: mempoolTx.acceleration || undefined,
+              }
             });
           }
         }
       }
 
       if (client['track-address']) {
-        const foundTransactions: TransactionExtended[] = [];
-
-        transactions.forEach((tx) => {
-          if (tx.vin && tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address'])) {
-            foundTransactions.push(tx);
-            return;
-          }
-          if (tx.vout && tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address'])) {
-            foundTransactions.push(tx);
-          }
-        });
+        const foundTransactions: TransactionExtended[] = Array.from(addressCache[client['track-address']]?.values() || []);
 
         if (foundTransactions.length) {
           foundTransactions.forEach((tx) => {
@@ -858,7 +856,7 @@ class WebsocketHandler {
         }
       }
 
-      if (client['track-mempool-block'] >= 0) {
+      if (client['track-mempool-block'] >= 0 && memPool.isInSync()) {
         const index = client['track-mempool-block'];
         if (mBlockDeltas && mBlockDeltas[index]) {
           response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, {
@@ -883,6 +881,52 @@ class WebsocketHandler {
         + '}';
   }
 
+  private makeAddressCache(transactions: MempoolTransactionExtended[]): { [address: string]: Set<MempoolTransactionExtended> } {
+    const addressCache: { [address: string]: Set<MempoolTransactionExtended> } = {};
+    for (const tx of transactions) {
+      for (const vin of tx.vin) {
+        if (vin?.prevout?.scriptpubkey_address) {
+          if (!addressCache[vin.prevout.scriptpubkey_address]) {
+            addressCache[vin.prevout.scriptpubkey_address] = new Set();
+          }
+          addressCache[vin.prevout.scriptpubkey_address].add(tx);
+        }
+        if (vin?.prevout?.scriptpubkey) {
+          if (!addressCache[vin.prevout.scriptpubkey]) {
+            addressCache[vin.prevout.scriptpubkey] = new Set();
+          }
+          addressCache[vin.prevout.scriptpubkey].add(tx);
+        }
+      }
+      for (const vout of tx.vout) {
+        if (vout?.scriptpubkey_address) {
+          if (!addressCache[vout?.scriptpubkey_address]) {
+            addressCache[vout?.scriptpubkey_address] = new Set();
+          }
+          addressCache[vout?.scriptpubkey_address].add(tx);
+        }
+        if (vout?.scriptpubkey) {
+          if (!addressCache[vout.scriptpubkey]) {
+            addressCache[vout.scriptpubkey] = new Set();
+          }
+          addressCache[vout.scriptpubkey].add(tx);
+        }
+      }
+    }
+    return addressCache;
+  }
+
+  private async getFullTransactions(transactions: MempoolTransactionExtended[]): Promise<MempoolTransactionExtended[]> {
+    for (let i = 0; i < transactions.length; i++) {
+      try {
+        transactions[i] = await transactionUtils.$getMempoolTransactionExtended(transactions[i].txid, true);
+      } catch (e) {
+        logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+      }
+    }
+    return transactions;
+  }
+
   private printLogs(): void {
     if (this.wss) {
       const count = this.wss?.clients?.size || 0;

backend/src/config.ts

@@ -5,13 +5,14 @@ const configFromFile = require(
 interface IConfig {
   MEMPOOL: {
     ENABLED: boolean;
-    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'regtest' | 'liquid' | 'liquidtestnet';
+    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
     SPAWN_CLUSTER_PROCS: number;
     API_URL_PREFIX: string;
     POLL_RATE_MS: number;
     CACHE_DIR: string;
+    CACHE_ENABLED: boolean;
     CLEAR_PROTECTION_MINUTES: number;
     RECOMMENDED_FEE_PERCENTILE: number;
     BLOCK_WEIGHT_UNITS: number;
@@ -37,11 +38,13 @@ interface IConfig {
     DISK_CACHE_BLOCK_INTERVAL: number;
     MAX_PUSH_TX_SIZE_WEIGHT: number;
     ALLOW_UNREACHABLE: boolean;
+    PRICE_UPDATES_PER_HOUR: number;
   };
   ESPLORA: {
     REST_API_URL: string;
     UNIX_SOCKET_PATH: string | void | null;
     RETRY_UNIX_SOCKET_AFTER: number;
+    FALLBACK: string[];
   };
   LIGHTNING: {
     ENABLED: boolean;
@@ -114,10 +117,6 @@ interface IConfig {
     USERNAME: string;
     PASSWORD: string;
   };
-  PRICE_DATA_SERVER: {
-    TOR_URL: string;
-    CLEARNET_URL: string;
-  };
   EXTERNAL_DATA_SERVER: {
     MEMPOOL_API: string;
     MEMPOOL_ONION: string;
@@ -137,7 +136,15 @@ interface IConfig {
     AUDIT: boolean;
     AUDIT_START_HEIGHT: number;
     SERVERS: string[];
-  }
+  },
+  MEMPOOL_SERVICES: {
+    API: string;
+    ACCELERATIONS: boolean;
+  },
+  REDIS: {
+    ENABLED: boolean;
+    UNIX_SOCKET_PATH: string;
+  },
 }
 
 const defaults: IConfig = {
@@ -150,6 +157,7 @@ const defaults: IConfig = {
     'API_URL_PREFIX': '/api/v1/',
     'POLL_RATE_MS': 2000,
     'CACHE_DIR': './cache',
+    'CACHE_ENABLED': true,
     'CLEAR_PROTECTION_MINUTES': 20,
     'RECOMMENDED_FEE_PERCENTILE': 50,
     'BLOCK_WEIGHT_UNITS': 4000000,
@@ -175,11 +183,13 @@ const defaults: IConfig = {
     'DISK_CACHE_BLOCK_INTERVAL': 6,
     'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
     'ALLOW_UNREACHABLE': true,
+    'PRICE_UPDATES_PER_HOUR': 1,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
     'UNIX_SOCKET_PATH': null,
     'RETRY_UNIX_SOCKET_AFTER': 30000,
+    'FALLBACK': [],
   },
   'ELECTRUM': {
     'HOST': '127.0.0.1',
@@ -252,10 +262,6 @@ const defaults: IConfig = {
     'USERNAME': '',
     'PASSWORD': ''
   },
-  'PRICE_DATA_SERVER': {
-    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
-    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
-  },
   'EXTERNAL_DATA_SERVER': {
     'MEMPOOL_API': 'https://mempool.space/api/v1',
     'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@@ -275,7 +281,15 @@ const defaults: IConfig = {
     'AUDIT': false,
     'AUDIT_START_HEIGHT': 774000,
     'SERVERS': [],
-  }
+  },
+  'MEMPOOL_SERVICES': {
+    'API': '',
+    'ACCELERATIONS': false,
+  },
+  'REDIS': {
+    'ENABLED': false,
+    'UNIX_SOCKET_PATH': '',
+  },
 };
 
 class Config implements IConfig {
@@ -292,10 +306,11 @@ class Config implements IConfig {
   LND: IConfig['LND'];
   CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
-  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
   REPLICATION: IConfig['REPLICATION'];
+  MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES'];
+  REDIS: IConfig['REDIS'];
 
   constructor() {
     const configs = this.merge(configFromFile, defaults);
@@ -312,10 +327,11 @@ class Config implements IConfig {
     this.LND = configs.LND;
     this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
-    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
     this.MAXMIND = configs.MAXMIND;
     this.REPLICATION = configs.REPLICATION;
+    this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES;
+    this.REDIS = configs.REDIS;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/index.ts

@@ -30,6 +30,7 @@ import generalLightningRoutes from './api/explorer/general.routes';
 import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
 import networkSyncService from './tasks/lightning/network-sync.service';
 import statisticsRoutes from './api/statistics/statistics.routes';
+import pricesRoutes from './api/prices/prices.routes';
 import miningRoutes from './api/mining/mining-routes';
 import bisqRoutes from './api/bisq/bisq.routes';
 import liquidRoutes from './api/liquid/liquid.routes';
@@ -41,6 +42,7 @@ import chainTips from './api/chain-tips';
 import { AxiosError } from 'axios';
 import v8 from 'v8';
 import { formatBytes, getBytesUnit } from './utils/format';
+import redisCache from './api/redis-cache';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -89,6 +91,10 @@ class Server {
   async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
+    if (config.MEMPOOL.BACKEND === 'esplora') {
+      bitcoinApi.startHealthChecks();
+    }
+
     if (config.DATABASE.ENABLED) {
       await DB.checkDbConnection();
       try {
@@ -122,7 +128,11 @@ class Server {
     await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
     if (config.MEMPOOL.ENABLED) {
-      await diskCache.$loadMempoolCache();
+      if (config.MEMPOOL.CACHE_ENABLED) {
+        await diskCache.$loadMempoolCache();
+      } else if (config.REDIS.ENABLED) {
+        await redisCache.$loadCache();
+      }
     }
 
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
@@ -183,14 +193,16 @@ class Server {
       }
       const newMempool = await bitcoinApi.$getRawMempool();
       const numHandledBlocks = await blocks.$updateBlocks();
+      const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1);
       if (numHandledBlocks === 0) {
-        await memPool.$updateMempool(newMempool);
+        await memPool.$updateMempool(newMempool, pollRate);
       }
       indexer.$run();
+      priceUpdater.$run();
 
       // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
       const elapsed = Date.now() - start;
-      const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed)
+      const remainingTime = Math.max(0, pollRate - elapsed);
       setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime);
       this.backendRetryCount = 0;
     } catch (e: any) {
@@ -255,6 +267,7 @@ class Server {
   
   setUpHttpApiRoutes(): void {
     bitcoinRoutes.initRoutes(this.app);
+    pricesRoutes.initRoutes(this.app);
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
       statisticsRoutes.initRoutes(this.app);
     }

backend/src/indexer.ts

@@ -105,6 +105,12 @@ class Indexer {
       return;
     }
 
+    try {
+      await priceUpdater.$run();
+    } catch (e) {
+      logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
     // Do not attempt to index anything unless Bitcoin Core is fully synced
     const blockchainInfo = await bitcoinClient.getBlockchainInfo();
     if (blockchainInfo.blocks !== blockchainInfo.headers) {
@@ -119,8 +125,6 @@ class Indexer {
     await this.checkAvailableCoreIndexes();
 
     try {
-      await priceUpdater.$run();
-
       const chainValid = await blocks.$generateBlockDatabase();
       if (chainValid === false) {
         // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration

backend/src/mempool.interfaces.ts

@@ -20,6 +20,7 @@ export interface PoolInfo {
   slug: string;
   avgMatchRate: number | null;
   avgFeeDelta: number | null;
+  poolUniqueId: number;
 }
 
 export interface PoolStats extends PoolInfo {
@@ -36,6 +37,7 @@ export interface BlockAudit {
   sigopTxs: string[],
   fullrbfTxs: string[],
   addedTxs: string[],
+  acceleratedTxs: string[],
   matchRate: number,
   expectedFees?: number,
   expectedWeight?: number,
@@ -91,6 +93,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
     block: number,
     vsize: number,
   };
+  acceleration?: boolean;
   uid?: number;
 }
 
@@ -182,6 +185,7 @@ export interface TransactionStripped {
   fee: number;
   vsize: number;
   value: number;
+  acc?: boolean;
   rate?: number; // effective fee rate
 }
 

backend/src/replication/AuditReplication.ts

@@ -116,6 +116,7 @@ class AuditReplication {
       freshTxs: auditSummary.freshTxs || [],
       sigopTxs: auditSummary.sigopTxs || [],
       fullrbfTxs: auditSummary.fullrbfTxs || [],
+      acceleratedTxs: auditSummary.acceleratedTxs || [],
       matchRate: auditSummary.matchRate,
       expectedFees: auditSummary.expectedFees,
       expectedWeight: auditSummary.expectedWeight,

backend/src/repositories/BlocksAuditsRepository.ts

@@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces';
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight)
-        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
-          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@@ -69,6 +69,7 @@ class BlocksAuditRepositories {
         fresh_txs as freshTxs,
         sigop_txs as sigopTxs,
         fullrbf_txs as fullrbfTxs,
+        accelerated_txs as acceleratedTxs,
         match_rate as matchRate,
         expected_fees as expectedFees,
         expected_weight as expectedWeight
@@ -83,6 +84,7 @@ class BlocksAuditRepositories {
         rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
         rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs);
         rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs);
+        rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs);
         rows[0].template = JSON.parse(rows[0].template);
 
         return rows[0];

backend/src/repositories/BlocksRepository.ts

@@ -1,3 +1,4 @@
+import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
 import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
@@ -12,6 +13,7 @@ import config from '../config';
 import chainTips from '../api/chain-tips';
 import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
+import transactionUtils from '../api/transaction-utils';
 
 interface DatabaseBlock {
   id: string;
@@ -539,7 +541,7 @@ class BlocksRepository {
    */
   public async $getBlocksDifficulty(): Promise<object[]> {
     try {
-      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
+      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks`);
       return rows;
     } catch (e) {
       logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e));
@@ -848,7 +850,7 @@ class BlocksRepository {
    */
   public async $getOldestConsecutiveBlock(): Promise<any> {
     try {
-      const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`);
+      const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, bits FROM blocks ORDER BY height DESC`);
       for (let i = 0; i < rows.length - 1; ++i) {
         if (rows[i].height - rows[i + 1].height > 1) {
           return rows[i];
@@ -1036,8 +1038,17 @@ class BlocksRepository {
     {
       extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
       if (extras.feePercentiles === null) {
-        const block = await bitcoinClient.getBlock(dbBlk.id, 2);
-        const summary = blocks.summarizeBlock(block);
+
+        let summary;
+        if (config.MEMPOOL.BACKEND === 'esplora') {
+          const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
+          summary = blocks.summarizeBlockTransactions(dbBlk.id, txs);
+        } else {
+          // Call Core RPC
+          const block = await bitcoinClient.getBlock(dbBlk.id, 2);
+          summary = blocks.summarizeBlock(block);
+        }
+
         await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
         extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
       }

backend/src/repositories/PoolsRepository.ts

@@ -40,7 +40,8 @@ class PoolsRepository {
           pools.link AS link,
           slug,
           AVG(blocks_audits.match_rate) AS avgMatchRate,
-          AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta
+          AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta,
+          unique_id as poolUniqueId
       FROM blocks
       JOIN pools on pools.id = pool_id
       LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
@@ -99,7 +100,7 @@ class PoolsRepository {
       if (parse) {
         rows[0].regexes = JSON.parse(rows[0].regexes);
       }
-      if (['testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
         rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
       } else if (parse) {
         rows[0].addresses = JSON.parse(rows[0].addresses);
@@ -131,7 +132,7 @@ class PoolsRepository {
       if (parse) {
         rows[0].regexes = JSON.parse(rows[0].regexes);
       }
-      if (['testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
         rows[0].addresses = []; // pools.json only contains mainnet addresses
       } else if (parse) {
         rows[0].addresses = JSON.parse(rows[0].addresses);

backend/src/tasks/pools-updater.ts

@@ -17,7 +17,7 @@ class PoolsUpdater {
   treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
   public async updatePoolsJson(): Promise<void> {
-    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) === false ||
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
       config.MEMPOOL.ENABLED === false
     ) {
       return;

backend/src/tasks/price-updater.ts

@@ -25,7 +25,10 @@ export interface PriceHistory {
 
 class PriceUpdater {
   public historyInserted = false;
-  private lastRun = 0;
+  private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
+  private cyclePosition = -1;
+  private firstRun = true;
+  private lastTime = -1;
   private lastHistoricalRun = 0;
   private running = false;
   private feeds: PriceFeed[] = [];
@@ -41,6 +44,8 @@ class PriceUpdater {
     this.feeds.push(new CoinbaseApi());
     this.feeds.push(new BitfinexApi());
     this.feeds.push(new GeminiApi());
+
+    this.setCyclePosition();
   }
 
   public getLatestPrices(): ApiPrice {
@@ -73,7 +78,7 @@ class PriceUpdater {
   }
 
   public async $run(): Promise<void> {
-    if (['testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+    if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') {
       // Coins have no value on testnet/signet, so we want to always show 0
       return;
     }
@@ -100,22 +105,48 @@ class PriceUpdater {
     this.running = false;
   }
 
+  private getMillisecondsSinceBeginningOfHour(): number {
+    const now = new Date();
+    const beginningOfHour = new Date(now);
+    beginningOfHour.setMinutes(0, 0, 0);
+    return now.getTime() - beginningOfHour.getTime();
+  }
+
+  private setCyclePosition(): void {
+    const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
+    for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) {
+      if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) {
+        this.cyclePosition = i;
+        return;
+      }
+    }
+    this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
+  }
+
   /**
    * Fetch last BTC price from exchanges, average them, and save it in the database once every hour
    */
   private async $updatePrice(): Promise<void> {
-    if (this.lastRun === 0 && config.DATABASE.ENABLED === true) {
-      this.lastRun = await PricesRepository.$getLatestPriceTime();
+    let forceUpdate = false;
+    if (this.firstRun === true && config.DATABASE.ENABLED === true) {
+      const lastUpdate = await PricesRepository.$getLatestPriceTime();
+      if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) {
+        forceUpdate = true;
+      }
+      this.firstRun = false;
     }
 
-    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) {
-      // Refresh only once every hour
+    const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
+
+    // Reset the cycle on new hour
+    if (this.lastTime > millisecondsSinceBeginningOfHour) {
+      this.cyclePosition = 0;
+    }
+    this.lastTime = millisecondsSinceBeginningOfHour;
+    if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate && this.cyclePosition !== 0) {
       return;
     }
 
-    const previousRun = this.lastRun;
-    this.lastRun = new Date().getTime() / 1000;
-
     for (const currency of this.currencies) {
       let prices: number[] = [];
 
@@ -146,26 +177,27 @@ class PriceUpdater {
       }
     }
 
-    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
-
-    if (config.DATABASE.ENABLED === true) {
+    if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) {
       // Save everything in db
       try {
         const p = 60 * 60 * 1000; // milliseconds in an hour
         const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
-        this.latestPrices.time = nowRounded.getTime() / 1000;
         await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
       } catch (e) {
-        this.lastRun = previousRun + 5 * 60;
         logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
       }
     }
 
+    this.latestPrices.time = Math.round(new Date().getTime() / 1000);
+    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
+
     if (this.ratesChangedCallback) {
       this.ratesChangedCallback(this.latestPrices);
     }
 
-    this.lastRun = new Date().getTime() / 1000;
+    if (!forceUpdate) {
+      this.cyclePosition++;
+    }
 
     if (this.latestPrices.USD === -1) {
       this.latestPrices = await PricesRepository.$getLatestConversionRates();

backend/src/utils/p-limit.ts

@@ -0,0 +1,179 @@
+/*
+MIT License
+
+Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be included in all copies
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+/*
+How it works:
+`this._head` is an instance of `Node` which keeps track of its current value and nests
+another instance of `Node` that keeps the value that comes after it. When a value is
+provided to `.enqueue()`, the code needs to iterate through `this._head`, going deeper
+and deeper to find the last value. However, iterating through every single item is slow.
+This problem is solved by saving a reference to the last value as `this._tail` so that
+it can reference it to add a new value.
+*/
+
+class Node {
+  value;
+  next;
+
+  constructor(value) {
+    this.value = value;
+  }
+}
+
+class Queue {
+  private _head;
+  private _tail;
+  private _size;
+
+  constructor() {
+    this.clear();
+  }
+
+  enqueue(value) {
+    const node = new Node(value);
+
+    if (this._head) {
+      this._tail.next = node;
+      this._tail = node;
+    } else {
+      this._head = node;
+      this._tail = node;
+    }
+
+    this._size++;
+  }
+
+  dequeue() {
+    const current = this._head;
+    if (!current) {
+      return;
+    }
+
+    this._head = this._head.next;
+    this._size--;
+    return current.value;
+  }
+
+  clear() {
+    this._head = undefined;
+    this._tail = undefined;
+    this._size = 0;
+  }
+
+  get size() {
+    return this._size;
+  }
+
+  *[Symbol.iterator]() {
+    let current = this._head;
+
+    while (current) {
+      yield current.value;
+      current = current.next;
+    }
+  }
+}
+
+interface LimitFunction {
+  readonly activeCount: number;
+  readonly pendingCount: number;
+  clearQueue: () => void;
+  <Arguments extends unknown[], ReturnType>(
+    fn: (...args: Arguments) => PromiseLike<ReturnType> | ReturnType,
+    ...args: Arguments
+  ): Promise<ReturnType>;
+}
+
+export default function pLimit(concurrency: number): LimitFunction {
+  if (
+    !(
+      (Number.isInteger(concurrency) ||
+        concurrency === Number.POSITIVE_INFINITY) &&
+      concurrency > 0
+    )
+  ) {
+    throw new TypeError('Expected `concurrency` to be a number from 1 and up');
+  }
+
+  const queue = new Queue();
+  let activeCount = 0;
+
+  const next = () => {
+    activeCount--;
+
+    if (queue.size > 0) {
+      queue.dequeue()();
+    }
+  };
+
+  const run = async (fn, resolve, args) => {
+    activeCount++;
+
+    const result = (async () => fn(...args))();
+
+    resolve(result);
+
+    try {
+      await result;
+    } catch {}
+
+    next();
+  };
+
+  const enqueue = (fn, resolve, args) => {
+    queue.enqueue(run.bind(undefined, fn, resolve, args));
+
+    (async () => {
+      // This function needs to wait until the next microtask before comparing
+      // `activeCount` to `concurrency`, because `activeCount` is updated asynchronously
+      // when the run function is dequeued and called. The comparison in the if-statement
+      // needs to happen asynchronously as well to get an up-to-date value for `activeCount`.
+      await Promise.resolve();
+
+      if (activeCount < concurrency && queue.size > 0) {
+        queue.dequeue()();
+      }
+    })();
+  };
+
+  const generator = (fn, ...args) =>
+    new Promise((resolve) => {
+      enqueue(fn, resolve, args);
+    });
+
+  Object.defineProperties(generator, {
+    activeCount: {
+      get: () => activeCount,
+    },
+    pendingCount: {
+      get: () => queue.size,
+    },
+    clearQueue: {
+      value: () => {
+        queue.clear();
+      },
+    },
+  });
+
+  return generator as any;
+}

contributors/Czino.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 29, 2023.
+
+Signed: Czino

contributors/andrewtoth.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 2, 2023.
+
+Signed: andrewtoth

contributors/bguillaumat.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: bguillaumat

contributors/fiatjaf.txt

@@ -0,0 +1,5 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+I also regret having ever contributed to this repository since they keep asking me to sign this legalese timewaste things.
+And finally I don't care about licenses and won't sue anyone over intellectual property, which is a fake statist construct invented by evil lobby lawyers.
+
+Signed: fiatjaf

contributors/rishkwal.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 29, 2023.
+
+Signed: rishkwal

docker/README.md

@@ -113,7 +113,8 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
-    "DISK_CACHE_BLOCK_INTERVAL": 6
+    "DISK_CACHE_BLOCK_INTERVAL": 6,
+    "PRICE_UPDATES_PER_HOUR": 1
   },
 ```
 
@@ -146,6 +147,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_CPFP_INDEXING: ""
       MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
       MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""
+      MEMPOOL_PRICE_UPDATES_PER_HOUR: ""
       ...
 ```
 
@@ -363,25 +365,6 @@ Corresponding `docker-compose.yml` overrides:
 
 <br/>
 
-`mempool-config.json`:
-```json
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
-    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
-  }
-```
-
-Corresponding `docker-compose.yml` overrides:
-```yaml
-  api:
-    environment:
-      PRICE_DATA_SERVER_TOR_URL: ""
-      PRICE_DATA_SERVER_CLEARNET_URL: ""
-      ...
-```
-
-<br/>
-
 `mempool-config.json`:
 ```json
   "LIGHTNING": {

docker/backend/mempool-config.json

@@ -8,6 +8,7 @@
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
     "POLL_RATE_MS": __MEMPOOL_POLL_RATE_MS__,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CACHE_ENABLED": __MEMPOOL_CACHE_ENABLED__,
     "CLEAR_PROTECTION_MINUTES": __MEMPOOL_CLEAR_PROTECTION_MINUTES__,
     "RECOMMENDED_FEE_PERCENTILE": __MEMPOOL_RECOMMENDED_FEE_PERCENTILE__,
     "BLOCK_WEIGHT_UNITS": __MEMPOOL_BLOCK_WEIGHT_UNITS__,
@@ -32,7 +33,8 @@
     "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__,
     "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__"
+    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -49,7 +51,8 @@
   "ESPLORA": {
     "REST_API_URL": "__ESPLORA_REST_API_URL__",
     "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
-    "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__
+    "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__,
+    "FALLBACK": __ESPLORA_FALLBACK__
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
@@ -110,10 +113,6 @@
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
     "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
   },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
-    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
     "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
@@ -133,5 +132,13 @@
     "AUDIT": __REPLICATION_AUDIT__,
     "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__,
     "SERVERS": __REPLICATION_SERVERS__
+  },
+  "MEMPOOL_SERVICES": {
+    "API": "__MEMPOOL_SERVICES_API__",
+    "ACCELERATIONS": __MEMPOOL_SERVICES_ACCELERATIONS__
+  },
+  "REDIS": {
+    "ENABLED": __REDIS_ENABLED__,
+    "UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__"
   }
 }

docker/backend/start.sh

@@ -9,6 +9,7 @@ __MEMPOOL_SPAWN_CLUSTER_PROCS__=${MEMPOOL_SPAWN_CLUSTER_PROCS:=0}
 __MEMPOOL_API_URL_PREFIX__=${MEMPOOL_API_URL_PREFIX:=/api/v1/}
 __MEMPOOL_POLL_RATE_MS__=${MEMPOOL_POLL_RATE_MS:=2000}
 __MEMPOOL_CACHE_DIR__=${MEMPOOL_CACHE_DIR:=./cache}
+__MEMPOOL_CACHE_ENABLED__=${MEMPOOL_CACHE_ENABLED:=true}
 __MEMPOOL_CLEAR_PROTECTION_MINUTES__=${MEMPOOL_CLEAR_PROTECTION_MINUTES:=20}
 __MEMPOOL_RECOMMENDED_FEE_PERCENTILE__=${MEMPOOL_RECOMMENDED_FEE_PERCENTILE:=50}
 __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000}
@@ -34,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
 __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000}
 __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true}
-
+__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -52,6 +53,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
 __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
 __ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"}
 __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000}
+__ESPLORA_FALLBACK__=${ESPLORA_FALLBACK:=[]}
 
 # SECOND_CORE_RPC
 __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
@@ -93,10 +95,6 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050}
 __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""}
 __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""}
 
-# PRICE_DATA_SERVER
-__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices}
-__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices}
-
 # EXTERNAL_DATA_SERVER
 __EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1}
 __EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1}
@@ -136,6 +134,13 @@ __REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true}
 __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000}
 __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]}
 
+# MEMPOOL_SERVICES
+__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""}
+__MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false}
+
+# REDIS
+__REDIS_ENABLED__=${REDIS_ENABLED:=true}
+__REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=true}
 
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
@@ -147,6 +152,7 @@ sed -i "s!__MEMPOOL_SPAWN_CLUSTER_PROCS__!${__MEMPOOL_SPAWN_CLUSTER_PROCS__}!g"
 sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POLL_RATE_MS__!${__MEMPOOL_POLL_RATE_MS__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CACHE_DIR__!${__MEMPOOL_CACHE_DIR__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_CACHE_ENABLED__!${__MEMPOOL_CACHE_ENABLED__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CLEAR_PROTECTION_MINUTES__!${__MEMPOOL_CLEAR_PROTECTION_MINUTES__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__!${__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_BLOCK_WEIGHT_UNITS__!${__MEMPOOL_BLOCK_WEIGHT_UNITS__}!g" mempool-config.json
@@ -165,13 +171,14 @@ sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-co
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_GBT__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json
 
 sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json
 sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json
@@ -186,6 +193,7 @@ sed -i "s!__ELECTRUM_TLS_ENABLED__!${__ELECTRUM_TLS_ENABLED__}!g" mempool-config
 sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json
 sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json
 sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json
+sed -i "s!__ESPLORA_FALLBACK__!${__ESPLORA_FALLBACK__}!g" mempool-config.json
 
 sed -i "s!__SECOND_CORE_RPC_HOST__!${__SECOND_CORE_RPC_HOST__}!g" mempool-config.json
 sed -i "s!__SECOND_CORE_RPC_PORT__!${__SECOND_CORE_RPC_PORT__}!g" mempool-config.json
@@ -221,9 +229,6 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json
 sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json
 sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json
 
-sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json
-sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json
-
 sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json
 sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json
 sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json
@@ -262,4 +267,12 @@ sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json
 sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json
 sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json
 
+# MEMPOOL_SERVICES
+sed -i "s!__MEMPOOL_SERVICES_API__!${__MEMPOOL_SERVICES_API__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS__}!g" mempool-config.json
+
+# REDIS
+sed -i "s!__REDIS_ENABLED__!${__REDIS_ENABLED__}!g" mempool-config.json
+sed -i "s!__REDIS_UNIX_SOCKET_PATH__!${__REDIS_UNIX_SOCKET_PATH__}!g" mempool-config.json
+
 node /backend/package/index.js

docker/frontend/entrypoint.sh

@@ -18,8 +18,7 @@ fi
 
 __TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
 __SIGNET_ENABLED__=${SIGNET_ENABLED:=false}
-__REGTEST_ENABLED__=${REGTEST_ENABLED:=false}
-__LIQUID_ENABLED__=${LIQUID_EANBLED:=false}
+__LIQUID_ENABLED__=${LIQUID_ENABLED:=false}
 __LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false}
 __BISQ_ENABLED__=${BISQ_ENABLED:=false}
 __BISQ_SEPARATE_BACKEND__=${BISQ_SEPARATE_BACKEND:=false}
@@ -45,7 +44,6 @@ __HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
 # Export as environment variables to be used by envsubst
 export __TESTNET_ENABLED__
 export __SIGNET_ENABLED__
-export __REGTEST_ENABLED__
 export __LIQUID_ENABLED__
 export __LIQUID_TESTNET_ENABLED__
 export __BISQ_ENABLED__

frontend/cypress/support/commands.ts

@@ -42,9 +42,6 @@
 // -- This will overwrite an existing command --
 // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })
 
-'use strict'
-
-import 'cypress-wait-until';
 import { PageIdleDetector } from './PageIdleDetector';
 import { mockWebSocket } from './websocket';
 

frontend/cypress/support/e2e.ts

@@ -14,6 +14,7 @@
 // ***********************************************************
 
 // When a command from ./commands is ready to use, import with `import './commands'` syntax
+import 'cypress-wait-until';
 import './commands';
 import failOnConsoleError from 'cypress-fail-on-console-error';
 

frontend/cypress/tsconfig.json

@@ -2,7 +2,7 @@
   "extends": "../tsconfig.json",
   "include": ["**/*.ts"],
   "compilerOptions": {
-    "types": ["cypress"],
+    "types": ["cypress", "node", "cypress-wait-until"],
     "lib": ["es2015", "dom"],
     "allowJs": true,
     "noEmit": true,

frontend/mempool-frontend-config.sample.json

@@ -1,7 +1,6 @@
 {
   "TESTNET_ENABLED": false,
   "SIGNET_ENABLED": false,
-  "REGTEST_ENABLED": false,
   "LIQUID_ENABLED": false,
   "LIQUID_TESTNET_ENABLED": false,
   "BISQ_ENABLED": false,
@@ -23,5 +22,6 @@
   "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "LIGHTNING": false,
-  "HISTORICAL_PRICE": true
+  "HISTORICAL_PRICE": true,
+  "ACCELERATOR": false
 }

frontend/package-lock.json

@@ -58,9 +58,10 @@
       },
       "optionalDependencies": {
         "@cypress/schematic": "^2.5.0",
-        "cypress": "^12.17.1",
+        "@types/cypress": "^1.1.3",
+        "cypress": "^12.17.2",
         "cypress-fail-on-console-error": "~4.0.3",
-        "cypress-wait-until": "^1.7.2",
+        "cypress-wait-until": "^2.0.0",
         "mock-socket": "~9.2.1",
         "start-server-and-test": "~2.0.0"
       }
@@ -3925,6 +3926,16 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/cypress": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@types/cypress/-/cypress-1.1.3.tgz",
+      "integrity": "sha512-OXe0Gw8LeCflkG1oPgFpyrYWJmEKqYncBsD/J0r17r0ETx/TnIGDNLwXt/pFYSYuYTpzcq1q3g62M9DrfsBL4g==",
+      "deprecated": "This is a stub types definition for cypress (https://cypress.io). cypress provides its own type definitions, so you don't need @types/cypress installed!",
+      "optional": true,
+      "dependencies": {
+        "cypress": "*"
+      }
+    },
     "node_modules/@types/eslint": {
       "version": "8.4.1",
       "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.1.tgz",
@@ -6641,9 +6652,9 @@
       "peer": true
     },
     "node_modules/cypress": {
-      "version": "12.17.1",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.1.tgz",
-      "integrity": "sha512-eKfBgO6t8waEyhegL4gxD7tcI6uTCGttu+ZU7y9Hq8BlpMztd7iLeIF4AJFAnbZH1xjX+wwgg4cRKFNSvv3VWQ==",
+      "version": "12.17.2",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.2.tgz",
+      "integrity": "sha512-hxWAaWbqQBzzMuadSGSuQg5PDvIGOovm6xm0hIfpCVcORsCAj/gF2p0EvfnJ4f+jK2PCiDgP6D2eeE9/FK4Mjg==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
@@ -6710,10 +6721,14 @@
       }
     },
     "node_modules/cypress-wait-until": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz",
-      "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==",
-      "optional": true
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-2.0.0.tgz",
+      "integrity": "sha512-ulUZyrWBn+OuC8oiQuGKAScDYfpaWnE3dEE/raUo64w4RHQxZrQ/iMIWT4ZjGMMPr3P+BFEALCRnjQeRqzZj6g==",
+      "optional": true,
+      "engines": {
+        "node": ">=18.16.0",
+        "npm": ">=9.5.1"
+      }
     },
     "node_modules/cypress/node_modules/@types/node": {
       "version": "14.18.53",
@@ -18862,6 +18877,15 @@
         "@types/node": "*"
       }
     },
+    "@types/cypress": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@types/cypress/-/cypress-1.1.3.tgz",
+      "integrity": "sha512-OXe0Gw8LeCflkG1oPgFpyrYWJmEKqYncBsD/J0r17r0ETx/TnIGDNLwXt/pFYSYuYTpzcq1q3g62M9DrfsBL4g==",
+      "optional": true,
+      "requires": {
+        "cypress": "*"
+      }
+    },
     "@types/eslint": {
       "version": "8.4.1",
       "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.1.tgz",
@@ -20968,9 +20992,9 @@
       "peer": true
     },
     "cypress": {
-      "version": "12.17.1",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.1.tgz",
-      "integrity": "sha512-eKfBgO6t8waEyhegL4gxD7tcI6uTCGttu+ZU7y9Hq8BlpMztd7iLeIF4AJFAnbZH1xjX+wwgg4cRKFNSvv3VWQ==",
+      "version": "12.17.2",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.2.tgz",
+      "integrity": "sha512-hxWAaWbqQBzzMuadSGSuQg5PDvIGOovm6xm0hIfpCVcORsCAj/gF2p0EvfnJ4f+jK2PCiDgP6D2eeE9/FK4Mjg==",
       "optional": true,
       "requires": {
         "@cypress/request": "^2.88.11",
@@ -21151,9 +21175,9 @@
       }
     },
     "cypress-wait-until": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz",
-      "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==",
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-2.0.0.tgz",
+      "integrity": "sha512-ulUZyrWBn+OuC8oiQuGKAScDYfpaWnE3dEE/raUo64w4RHQxZrQ/iMIWT4ZjGMMPr3P+BFEALCRnjQeRqzZj6g==",
       "optional": true
     },
     "d": {

frontend/package.json

@@ -110,9 +110,10 @@
   },
   "optionalDependencies": {
     "@cypress/schematic": "^2.5.0",
-    "cypress": "^12.17.1",
+    "@types/cypress": "^1.1.3",
+    "cypress": "^12.17.2",
     "cypress-fail-on-console-error": "~4.0.3",
-    "cypress-wait-until": "^1.7.2",
+    "cypress-wait-until": "^2.0.0",
     "mock-socket": "~9.2.1",
     "start-server-and-test": "~2.0.0"
   },

frontend/src/app/app-routing.module.ts

@@ -249,115 +249,6 @@ let routes: Routes = [
       },
     ]
   },
-  {
-    path: 'regtest',
-    children: [
-      {
-        path: 'mining/blocks',
-        redirectTo: 'blocks',
-        pathMatch: 'full'
-      },
-      {
-        path: '',
-        pathMatch: 'full',
-        loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule)
-      },
-      {
-        path: '',
-        component: MasterPageComponent,
-        children: [
-          {
-            path: 'tx/push',
-            component: PushTransactionComponent,
-          },
-          {
-            path: 'about',
-            component: AboutComponent,
-          },
-          {
-            path: 'blocks',
-            component: BlocksList,
-          },
-          {
-            path: 'rbf',
-            component: RbfList,
-          },
-          {
-            path: 'terms-of-service',
-            component: TermsOfServiceComponent
-          },
-          {
-            path: 'privacy-policy',
-            component: PrivacyPolicyComponent
-          },
-          {
-            path: 'trademark-policy',
-            component: TrademarkPolicyComponent
-          },
-          {
-            path: 'address/:id',
-            children: [],
-            component: AddressComponent,
-            data: {
-              ogImage: true,
-              networkSpecific: true,
-            }
-          },
-          {
-            path: 'tx',
-            data: { networkSpecific: true },
-            component: StartComponent,
-            children: [
-              {
-                path: ':id',
-                component: TransactionComponent
-              },
-            ],
-          },
-          {
-            path: 'block',
-            data: { networkSpecific: true },
-            component: StartComponent,
-            children: [
-              {
-                path: ':id',
-                component: BlockComponent,
-                data: {
-                  ogImage: true
-                }
-              },
-            ],
-          },
-          {
-            path: 'docs',
-            loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule)
-          },
-          {
-            path: 'api',
-            loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule)
-          },
-          {
-            path: 'lightning',
-            data: { networks: ['bitcoin'] },
-            loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule)
-          },
-        ],
-      },
-      {
-        path: 'status',
-        data: { networks: ['bitcoin', 'liquid'] },
-        component: StatusViewComponent
-      },
-      {
-        path: '',
-        loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule)
-      },
-      {
-        path: '**',
-        redirectTo: '/signet'
-      },
-    ]
-  },
   {
     path: '',
     pathMatch: 'full',

frontend/src/app/bisq/bisq-address/bisq-address.component.ts

@@ -47,6 +47,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
               catchError((err) => {
                 this.isLoadingAddress = false;
                 this.error = err;
+                this.seoService.logSoft404();
                 console.log(err);
                 return of(null);
               })
@@ -62,6 +63,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAddress = false;
       });
   }

frontend/src/app/bisq/bisq-block/bisq-block.component.ts

@@ -82,6 +82,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
       )
       .subscribe((block: BisqBlock) => {
         if (!block) {
+          this.seoService.logSoft404();
           return;
         }
         this.isLoading = false;
@@ -97,6 +98,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
 
   caughtHttpError(err: HttpErrorResponse){
     this.error = err;
+    this.seoService.logSoft404();
     return of(null);
   }
 }

frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts

@@ -70,11 +70,13 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
                     catchError((txError: HttpErrorResponse) => {
                       console.log(txError);
                       this.error = txError;
+                      this.seoService.logSoft404();
                       return of(null);
                     })
                   );
               }
               this.error = bisqTxError;
+              this.seoService.logSoft404();
               return of(null);
             })
           );
@@ -103,6 +105,7 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
       this.isLoadingTx = false;
 
       if (!tx) {
+        this.seoService.logSoft404();
         return;
       }
 

frontend/src/app/bitcoin.utils.ts

@@ -271,11 +271,6 @@ const featureActivation = {
     segwit: 0,
     taproot: 0,
   },
-  regtest: {
-    rbf: 0,
-    segwit: 0,
-    taproot: 0,
-  },
 };
 
 export function isFeatureActive(network: string, height: number, feature: 'rbf' | 'segwit' | 'taproot'): boolean {

frontend/src/app/components/about/about.component.html

@@ -31,6 +31,14 @@
     <track label="Português" kind="captions" srclang="pt" src="/resources/promo-video/pt.vtt" [attr.default]="showSubtitles('pt') ? '' : null">
   </video>
 
+  <ng-container *ngIf="false && officialMempoolSpace">
+    <h3 class="mt-5">Sponsor the project</h3>
+    <div class="d-flex justify-content-center" style="max-width: 90%; margin: 35px auto 75px auto; column-gap: 15px">
+      <a href="/sponsor" class="btn" style="background-color: rgba(152, 88, 255, 0.75); box-shadow: 0px 0px 50px 5px rgba(152, 88, 255, 0.75)" i18n="about.community-sponsor-button">Community</a>
+      <a href="/enterprise" class="btn" style="background-color: rgba(152, 88, 255, 0.75); box-shadow: 0px 0px 50px 5px rgba(152, 88, 255, 0.75)" i18n="about.enterprise-sponsor-button">Enterprise</a>
+    </div>
+  </ng-container>
+
   <div class="enterprise-sponsor" id="enterprise-sponsors">
     <h3 i18n="about.sponsors.enterprise.withRocket">Enterprise Sponsors 🚀</h3>
     <div class="wrapper">
@@ -173,34 +181,44 @@
         </svg>
         <span>Exodus</span>
       </a>
-      <a href="https://www.luminex.io" target="_blank" title="Luminex">
-        <svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" width="66.95" height="80" viewBox="0 0 300.43 385" style="padding-top: 10px;">
-          <defs>
-            <style>
-              .lum-cls-1 {
-                fill: #f2ea25;
-              }
-            </style>
-          </defs>
-          <path class="lum-cls-1" d="m309.02,90.04c0,49.65-38.73,90.04-95.34,90.04s-95.34-40.39-95.34-90.04S153.77,0,213.69,0c56.28,0,95.34,40.39,95.34,90.04Zm-63.56,0c0-20.52-14.23-37.07-31.78-37.07s-31.78,16.55-31.78,37.07,14.23,37.07,31.78,37.07,31.78-16.55,31.78-37.07Z"/>
-          <path class="lum-cls-1" d="m311.87,372.67h-66.34l-31.84-47.76-31.84,47.76h-66.34l58.38-90.22-53.07-79.61h66.34l26.54,42.46,26.53-42.46h66.34l-53.07,79.61,58.38,90.22Z"/>
-          <rect class="lum-cls-1" width="60.69" height="372.67"/>
-        </svg>
-        <span>Luminex</span>
-      </a>
     </div>
   </div>
 
-  <div class="community-sponsor" id="community-sponsors">
-    <h3 i18n="about.sponsors.withHeart">Community Sponsors ❤️</h3>
+  <ng-container *ngIf="officialMempoolSpace">
+    <div *ngIf="profiles$ | async as profiles" id="community-sponsors">
+      <div class="community-sponsor" style="margin-bottom: 68px" *ngIf="profiles.whales.length > 0">
+        <h3 i18n="about.sponsors.withHeart">Whale Sponsors</h3>
+        <div class="wrapper">
+          <ng-container>
+            <ng-template ngFor let-sponsor [ngForOf]="profiles.whales">
+              <a [href]="'https://twitter.com/' + sponsor.username" target="_blank" rel="sponsored" [title]="sponsor.username">
+                <img class="image" [src]="'data:' + sponsor.image_mime + ';base64,' + sponsor.image" onError="this.src = '/resources/profile/unknown.svg'; this.className = 'image unknown'"/>
+              </a>
+            </ng-template>
+          </ng-container>
+        </div>
+      </div>
 
+      <div class="community-sponsor" style="margin-bottom: 68px" *ngIf="profiles.chads.length > 0">
+        <h3 i18n="about.sponsors.withHeart">Chad Sponsors</h3>
+        <div class="wrapper">
+          <ng-template ngFor let-sponsor [ngForOf]="profiles.chads">
+            <a [href]="'https://twitter.com/' + sponsor.username" target="_blank" rel="sponsored" [title]="sponsor.username">
+              <img class="image" [src]="'data:' + sponsor.image_mime + ';base64,' + sponsor.image" onError="this.src = '/resources/profile/unknown.svg'; this.className = 'image unknown'"/>
+            </a>
+          </ng-template>
+        </div>
+      </div>
+    </div>
+  </ng-container>
+
+  <div class="community-sponsor" style="margin-bottom: 68px">
+    <h3 i18n="about.sponsors.withHeart">OG Sponsors ❤️</h3>
     <div class="wrapper">
-      <ng-container *ngIf="sponsors$ | async as sponsors; else loadingSponsors">
-        <ng-template ngFor let-sponsor [ngForOf]="sponsors">
-          <a [href]="'https://twitter.com/' + sponsor.handle" target="_blank" rel="sponsored" [title]="sponsor.handle">
-            <img class="image" [src]="'/api/v1/donations/images/' + sponsor.handle" />
-          </a>
-        </ng-template>
+      <ng-container *ngIf="ogs$ | async as ogs; else loadingSponsors">
+        <a *ngFor="let ogSponsor of ogs" [href]="'https://twitter.com/' + ogSponsor.handle" target="_blank" rel="sponsored" [title]="ogSponsor.handle">
+          <img class="image" [src]="'/api/v1/donations/images/' + ogSponsor.handle" onError="this.src = '/resources/profile/unknown.svg'; this.className = 'image unknown'"/>
+        </a>
       </ng-container>
     </div>
   </div>
@@ -340,7 +358,7 @@
       <div class="wrapper">
         <ng-template ngFor let-translator [ngForOf]="translators">
           <a [href]="'https://twitter.com/' + translator.value" target="_blank" [title]="translator.key">
-            <img class="image" [src]="'/api/v1/translators/images/' + translator.value" />
+            <img class="image" [src]="'/api/v1/translators/images/' + translator.value" onError="this.src = '/resources/profile/unknown.svg'; this.className = 'image unknown'"/>
           </a>
         </ng-template>
       </div>
@@ -354,7 +372,7 @@
       <div class="wrapper">
         <ng-template ngFor let-contributor [ngForOf]="contributors.regular">
           <a [href]="'https://github.com/' + contributor.name" target="_blank" [title]="contributor.name">
-            <img class="image" [src]="'/api/v1/contributors/images/' + contributor.id" />
+            <img class="image" [src]="'/api/v1/contributors/images/' + contributor.id" onError="this.src = '/resources/profile/unknown.svg'; this.className = 'image unknown'"/>
             <span>{{ contributor.name }}</span>
           </a>
         </ng-template>
@@ -366,7 +384,7 @@
       <div class="wrapper">
         <ng-template ngFor let-contributor [ngForOf]="contributors.core">
           <a [href]="'https://github.com/' + contributor.name" target="_blank" [title]="contributor.name">
-            <img class="image" [src]="'/api/v1/contributors/images/' + contributor.id" />
+            <img class="image" [src]="'/api/v1/contributors/images/' + contributor.id" onError="this.src = '/resources/profile/unknown.svg'; this.className = 'image unknown'"/>
             <span>{{ contributor.name }}</span>
           </a>
         </ng-template>
@@ -411,7 +429,7 @@
       Trademark Notice<br>
     </div>
     <p>
-      The Mempool Open Source Project&trade;, mempool.space&trade;, the mempool logo&reg;, the mempool.space logos&trade;, the mempool square logo&reg;, and the mempool blocks logo&trade; are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries.
+      The Mempool Open Source Project&reg;, mempool.space&trade;, the mempool logo&reg;, the mempool.space logos&trade;, the mempool square logo&reg;, and the mempool blocks logo&trade; are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries.
     </p>
     <p>
       While our software is available under an open source software license, the copyright license does not include an implied right or license to use our trademarks. See our <a href="https://mempool.space/trademark-policy">Trademark Policy and Guidelines</a> for more details, published on &lt;https://mempool.space/trademark-policy&gt;.

frontend/src/app/components/about/about.component.scss

@@ -10,6 +10,9 @@
     margin: 25px;
     line-height: 32px;
   }
+  .unknown {
+    border: 1px solid #b4b4b4;
+  }
 
   .image.not-rounded {
     border-radius: 0;

frontend/src/app/components/about/about.component.ts

@@ -6,7 +6,7 @@ import { Observable } from 'rxjs';
 import { ApiService } from '../../services/api.service';
 import { IBackendInfo } from '../../interfaces/websocket.interface';
 import { Router, ActivatedRoute } from '@angular/router';
-import { map, tap } from 'rxjs/operators';
+import { map, share, tap } from 'rxjs/operators';
 import { ITranslators } from '../../interfaces/node-api.interface';
 import { DOCUMENT } from '@angular/common';
 
@@ -19,14 +19,16 @@ import { DOCUMENT } from '@angular/common';
 export class AboutComponent implements OnInit {
   @ViewChild('promoVideo') promoVideo: ElementRef;
   backendInfo$: Observable<IBackendInfo>;
-  sponsors$: Observable<any>;
-  translators$: Observable<ITranslators>;
-  allContributors$: Observable<any>;
   frontendGitCommitHash = this.stateService.env.GIT_COMMIT_HASH;
   packetJsonVersion = this.stateService.env.PACKAGE_JSON_VERSION;
   officialMempoolSpace = this.stateService.env.OFFICIAL_MEMPOOL_SPACE;
   showNavigateToSponsor = false;
 
+  profiles$: Observable<any>;
+  translators$: Observable<ITranslators>;
+  allContributors$: Observable<any>;
+  ogs$: Observable<any>;
+
   constructor(
     private websocketService: WebsocketService,
     private seoService: SeoService,
@@ -43,10 +45,13 @@ export class AboutComponent implements OnInit {
     this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`);
     this.websocketService.want(['blocks']);
 
-    this.sponsors$ = this.apiService.getDonation$()
-      .pipe(
-        tap(() => this.goToAnchor())
-      );
+    this.profiles$ = this.apiService.getAboutPageProfiles$().pipe(
+      tap(() => {
+        this.goToAnchor()
+      }),
+      share(),
+    )
+
     this.translators$ = this.apiService.getTranslators$()
       .pipe(
         map((translators) => {
@@ -59,6 +64,9 @@ export class AboutComponent implements OnInit {
         }),
         tap(() => this.goToAnchor())
       );
+
+    this.ogs$ = this.apiService.getOgs$();
+
     this.allContributors$ = this.apiService.getContributor$().pipe(
       map((contributors) => {
         return {

frontend/src/app/components/address/address-preview.component.ts

@@ -64,12 +64,12 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
           this.address = null;
           this.addressInfo = null;
           this.addressString = params.get('id') || '';
-          if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|[A-F0-9]{130}$/.test(this.addressString)) {
+          if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}$/.test(this.addressString)) {
             this.addressString = this.addressString.toLowerCase();
           }
           this.seoService.setTitle($localize`:@@address.component.browser-title:Address: ${this.addressString}:INTERPOLATION:`);
 
-          return (this.addressString.match(/[a-f0-9]{130}/)
+          return (this.addressString.match(/04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}/)
               ? this.electrsApiService.getPubKeyAddress$(this.addressString)
               : this.electrsApiService.getAddress$(this.addressString)
             ).pipe(

frontend/src/app/components/address/address.component.ts

@@ -72,7 +72,7 @@ export class AddressComponent implements OnInit, OnDestroy {
           this.addressInfo = null;
           document.body.scrollTo(0, 0);
           this.addressString = params.get('id') || '';
-          if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|[A-F0-9]{130}$/.test(this.addressString)) {
+          if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}$/.test(this.addressString)) {
             this.addressString = this.addressString.toLowerCase();
           }
           this.seoService.setTitle($localize`:@@address.component.browser-title:Address: ${this.addressString}:INTERPOLATION:`);
@@ -84,13 +84,14 @@ export class AddressComponent implements OnInit, OnDestroy {
           )
           .pipe(
             switchMap(() => (
-              this.addressString.match(/[a-f0-9]{130}/)
+              this.addressString.match(/04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}/)
               ? this.electrsApiService.getPubKeyAddress$(this.addressString)
               : this.electrsApiService.getAddress$(this.addressString)
             ).pipe(
                 catchError((err) => {
                   this.isLoadingAddress = false;
                   this.error = err;
+                  this.seoService.logSoft404();
                   console.log(err);
                   return of(null);
                 })
@@ -118,7 +119,7 @@ export class AddressComponent implements OnInit, OnDestroy {
           this.isLoadingAddress = false;
           this.isLoadingTransactions = true;
           return address.is_pubkey
-              ? this.electrsApiService.getScriptHashTransactions$('41' + address.address + 'ac')
+              ? this.electrsApiService.getScriptHashTransactions$((address.address.length === 66 ? '21' : '41') + address.address + 'ac')
               : this.electrsApiService.getAddressTransactions$(address.address);
         }),
         switchMap((transactions) => {
@@ -162,35 +163,13 @@ export class AddressComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAddress = false;
       });
 
     this.stateService.mempoolTransactions$
-      .subscribe((transaction) => {
-        if (this.transactions.some((t) => t.txid === transaction.txid)) {
-          return;
-        }
-
-        this.transactions.unshift(transaction);
-        this.transactions = this.transactions.slice();
-        this.txCount++;
-
-        if (transaction.vout.some((vout) => vout.scriptpubkey_address === this.address.address)) {
-          this.audioService.playSound('cha-ching');
-        } else {
-          this.audioService.playSound('chime');
-        }
-
-        transaction.vin.forEach((vin) => {
-          if (vin.prevout.scriptpubkey_address === this.address.address) {
-            this.sent += vin.prevout.value;
-          }
-        });
-        transaction.vout.forEach((vout) => {
-          if (vout.scriptpubkey_address === this.address.address) {
-            this.received += vout.value;
-          }
-        });
+      .subscribe(tx => {
+        this.addTransaction(tx);
       });
 
     this.stateService.blockTransactions$
@@ -200,12 +179,47 @@ export class AddressComponent implements OnInit, OnDestroy {
           tx.status = transaction.status;
           this.transactions = this.transactions.slice();
           this.audioService.playSound('magic');
+        } else {
+          if (this.addTransaction(transaction, false)) {
+            this.audioService.playSound('magic');
+          }
         }
         this.totalConfirmedTxCount++;
         this.loadedConfirmedTxCount++;
       });
   }
 
+  addTransaction(transaction: Transaction, playSound: boolean = true): boolean {
+    if (this.transactions.some((t) => t.txid === transaction.txid)) {
+      return false;
+    }
+
+    this.transactions.unshift(transaction);
+    this.transactions = this.transactions.slice();
+    this.txCount++;
+
+    if (playSound) {
+      if (transaction.vout.some((vout) => vout?.scriptpubkey_address === this.address.address)) {
+        this.audioService.playSound('cha-ching');
+      } else {
+        this.audioService.playSound('chime');
+      }
+    }
+
+    transaction.vin.forEach((vin) => {
+      if (vin?.prevout?.scriptpubkey_address === this.address.address) {
+        this.sent += vin.prevout.value;
+      }
+    });
+    transaction.vout.forEach((vout) => {
+      if (vout?.scriptpubkey_address === this.address.address) {
+        this.received += vout.value;
+      }
+    });
+
+    return true;
+  }
+
   loadMore() {
     if (this.isLoadingTransactions || !this.totalConfirmedTxCount || this.loadedConfirmedTxCount >= this.totalConfirmedTxCount) {
       return;

frontend/src/app/components/asset/asset.component.ts

@@ -86,6 +86,7 @@ export class AssetComponent implements OnInit, OnDestroy {
                   catchError((err) => {
                     this.isLoadingAsset = false;
                     this.error = err;
+                    this.seoService.logSoft404();
                     console.log(err);
                     return of(null);
                   })
@@ -153,6 +154,7 @@ export class AssetComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAsset = false;
       });
 

frontend/src/app/components/bisq-master-page/bisq-master-page.component.html

@@ -39,7 +39,7 @@
     </ng-container>
   </a>
 
-  <div ngbDropdown (window:resize)="onResize($event)" class="dropdown-container" *ngIf="env.TESTNET_ENABLED || env.SIGNET_ENABLED || env.REGTEST_ENABLED || env.LIQUID_ENABLED || env.BISQ_ENABLED || env.LIQUID_TESTNET_ENABLED">
+  <div ngbDropdown (window:resize)="onResize($event)" class="dropdown-container" *ngIf="env.TESTNET_ENABLED || env.SIGNET_ENABLED || env.LIQUID_ENABLED || env.BISQ_ENABLED || env.LIQUID_TESTNET_ENABLED">
     <button ngbDropdownToggle type="button" class="btn btn-secondary dropdown-toggle-split d-flex justify-content-center align-items-center" aria-haspopup="true">
       <app-svg-images class="d-flex justify-content-center align-items-center current-network-svg" name="bisq" width="20" height="20" viewBox="0 0 80 80"></app-svg-images>
     </button>
@@ -47,7 +47,6 @@
       <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + (networkPaths['mainnet'] || '/')" ngbDropdownItem class="mainnet"><app-svg-images name="bitcoin" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Mainnet</a>
       <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + (networkPaths['signet'] || '/signet')" ngbDropdownItem *ngIf="env.SIGNET_ENABLED" class="signet"><app-svg-images name="signet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Signet</a>
       <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + (networkPaths['testnet'] || '/testnet')" ngbDropdownItem *ngIf="env.TESTNET_ENABLED" class="testnet"><app-svg-images name="testnet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Testnet</a>
-      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + (networkPaths['regtest'] || '/regtest')" ngbDropdownItem *ngIf="env.REGTEST_ENABLED" class="regtest"><app-svg-images name="regtest" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Regtest</a>
       <h6 class="dropdown-header" i18n="master-page.layer2-networks-header">Layer 2 Networks</h6>
       <a ngbDropdownItem class="mainnet active" [routerLink]="networkPaths['bisq'] || '/'"><app-svg-images name="bisq" width="20" height="20" viewBox="0 0 75 75" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Bisq</a>
       <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage + (networkPaths['liquid'] || '/')" ngbDropdownItem *ngIf="env.LIQUID_ENABLED" class="liquid"><app-svg-images name="liquid" width="22" height="22" viewBox="0 0 125 125" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Liquid</a>

frontend/src/app/components/bisq-master-page/bisq-master-page.component.scss

@@ -120,10 +120,6 @@ nav {
   background-color: #6f1d5d;
 }
 
-.regtest.active {
-  background-color: #a5a5a5;
-}
-
 .dropdown-divider {
   border-top: 1px solid #121420;
 }

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss

@@ -25,7 +25,8 @@
   flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  height: calc(100vh - 250px);
+  height: calc(100vh - 225px);
+  min-height: 400px;
   @media (min-width: 992px) {
     height: calc(100vh - 150px);
   }

frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss

@@ -25,7 +25,8 @@
   flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  height: calc(100vh - 250px);
+  height: calc(100vh - 225px);
+  min-height: 400px;
   @media (min-width: 992px) {
     height: calc(100vh - 150px);
   }

frontend/src/app/components/block-health-graph/block-health-graph.component.scss

@@ -25,7 +25,8 @@
   flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  height: calc(100vh - 250px);
+  height: calc(100vh - 225px);
+  min-height: 400px;
   @media (min-width: 992px) {
     height: calc(100vh - 150px);
   }

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     }
   }
 
-  update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
+  update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
     if (this.scene) {
       this.scene.update(add, remove, change, direction, resetLayout);
       this.start();

frontend/src/app/components/block-overview-graph/block-scene.ts

@@ -150,7 +150,7 @@ export default class BlockScene {
     this.updateAll(startTime, 200, direction);
   }
 
-  update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
+  update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
     const startTime = performance.now();
     const removed = this.removeBatch(remove, startTime, direction);
 
@@ -175,6 +175,7 @@ export default class BlockScene {
       // update effective rates
       change.forEach(tx => {
         if (this.txs[tx.txid]) {
+          this.txs[tx.txid].acc = tx.acc;
           this.txs[tx.txid].feerate = tx.rate || (this.txs[tx.txid].fee / this.txs[tx.txid].vsize);
           this.txs[tx.txid].rate = tx.rate;
           this.txs[tx.txid].dirty = true;

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -17,6 +17,7 @@ const auditColors = {
   missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7),
   added: hexToColor('0099ff'),
   selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7),
+  accelerated: hexToColor('8F5FF6'),
 };
 
 // convert from this class's update format to TxSprite's update format
@@ -37,8 +38,9 @@ export default class TxView implements TransactionStripped {
   vsize: number;
   value: number;
   feerate: number;
+  acc?: boolean;
   rate?: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated';
   context?: 'projected' | 'actual';
   scene?: BlockScene;
 
@@ -63,6 +65,7 @@ export default class TxView implements TransactionStripped {
     this.vsize = tx.vsize;
     this.value = tx.value;
     this.feerate = tx.rate || (tx.fee / tx.vsize); // sort by effective fee rate where available
+    this.acc = tx.acc;
     this.rate = tx.rate;
     this.status = tx.status;
     this.initialised = false;
@@ -199,6 +202,11 @@ export default class TxView implements TransactionStripped {
     const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1];
     // Normal mode
     if (!this.scene?.highlightingEnabled) {
+      if (this.acc) {
+        return auditColors.accelerated;
+      } else {
+        return feeLevelColor;
+      }
       return feeLevelColor;
     }
     // Block audit
@@ -207,7 +215,7 @@ export default class TxView implements TransactionStripped {
         return auditColors.censored;
       case 'missing':
       case 'sigop':
-      case 'fullrbf':
+      case 'rbf':
         return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1];
       case 'fresh':
       case 'freshcpfp':
@@ -216,6 +224,8 @@ export default class TxView implements TransactionStripped {
         return auditColors.added;
       case 'selected':
         return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1];
+      case 'accelerated':
+        return auditColors.accelerated;
       case 'found':
         if (this.context === 'projected') {
           return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1];
@@ -223,7 +233,11 @@ export default class TxView implements TransactionStripped {
           return feeLevelColor;
         }
       default:
-        return feeLevelColor;
+        if (this.acc) {
+          return auditColors.accelerated;
+        } else {
+          return feeLevelColor;
+        }
     }
   }
 }

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -29,7 +29,8 @@
         </td>
       </tr>
       <tr *ngIf="effectiveRate && effectiveRate !== feeRate">
-        <td class="td-width" i18n="transaction.effective-fee-rate|Effective transaction fee rate">Effective fee rate</td>
+        <td *ngIf="!this.acceleration" class="td-width" i18n="transaction.effective-fee-rate|Effective transaction fee rate">Effective fee rate</td>
+        <td *ngIf="this.acceleration" class="td-width" i18n="transaction.effective-fee-rate|Effective transaction fee rate">Accelerated fee rate</td>
         <td>
           <app-fee-rate [fee]="effectiveRate"></app-fee-rate>
         </td>
@@ -53,7 +54,8 @@
           <td *ngSwitchCase="'freshcpfp'"><span class="badge badge-warning" i18n="transaction.audit.recently-cpfped">Recently CPFP'd</span></td>
           <td *ngSwitchCase="'added'"><span class="badge badge-warning" i18n="transaction.audit.added">Added</span></td>
           <td *ngSwitchCase="'selected'"><span class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span></td>
-          <td *ngSwitchCase="'fullrbf'"><span class="badge badge-warning" i18n="transaction.audit.fullrbf">Full RBF</span></td>
+          <td *ngSwitchCase="'rbf'"><span class="badge badge-warning" i18n="transaction.audit.conflicting">Conflicting</span></td>
+          <td *ngSwitchCase="'accelerated'"><span class="badge badge-success" i18n="transaction.audit.accelerated">Accelerated</span></td>
         </ng-container>
       </tr>
     </tbody>

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts

@@ -21,6 +21,7 @@ export class BlockOverviewTooltipComponent implements OnChanges {
   vsize = 1;
   feeRate = 0;
   effectiveRate;
+  acceleration;
 
   tooltipPosition: Position = { x: 0, y: 0 };
 
@@ -53,6 +54,7 @@ export class BlockOverviewTooltipComponent implements OnChanges {
       this.vsize = tx.vsize || 1;
       this.feeRate = this.fee / this.vsize;
       this.effectiveRate = tx.rate;
+      this.acceleration = tx.acc;
     }
   }
 }

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss

@@ -25,7 +25,8 @@
   flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  height: calc(100vh - 250px);
+  height: calc(100vh - 225px);
+  min-height: 400px;
   @media (min-width: 992px) {
     height: calc(100vh - 150px);
   }

frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss

@@ -25,7 +25,8 @@
   flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  height: calc(100vh - 250px);
+  height: calc(100vh - 225px);
+  min-height: 400px;
   @media (min-width: 992px) {
     height: calc(100vh - 150px);
   }

frontend/src/app/components/block/block-preview.component.ts

@@ -82,6 +82,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
               }),
               catchError((err) => {
                 this.error = err;
+                this.seoService.logSoft404();
                 this.openGraphService.fail('block-data-' + this.rawId);
                 this.openGraphService.fail('block-viz-' + this.rawId);
                 return of(null);
@@ -138,6 +139,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
     (error) => {
       this.error = error;
       this.isLoadingOverview = false;
+      this.seoService.logSoft404();
       this.openGraphService.fail('block-viz-' + this.rawId);
       this.openGraphService.fail('block-data-' + this.rawId);
       if (this.blockGraph) {

frontend/src/app/components/block/block.component.ts

@@ -206,6 +206,7 @@ export class BlockComponent implements OnInit, OnDestroy {
                       this.error = err;
                       this.isLoadingBlock = false;
                       this.isLoadingOverview = false;
+                      this.seoService.logSoft404();
                       return EMPTY;
                     })
                   );
@@ -214,6 +215,7 @@ export class BlockComponent implements OnInit, OnDestroy {
                   this.error = err;
                   this.isLoadingBlock = false;
                   this.isLoadingOverview = false;
+                  this.seoService.logSoft404();
                   return EMPTY;
                 }),
               );
@@ -229,6 +231,7 @@ export class BlockComponent implements OnInit, OnDestroy {
               this.error = err;
               this.isLoadingBlock = false;
               this.isLoadingOverview = false;
+              this.seoService.logSoft404();
               return EMPTY;
             })
           );
@@ -339,13 +342,17 @@ export class BlockComponent implements OnInit, OnDestroy {
         const isSelected = {};
         const isFresh = {};
         const isSigop = {};
-        const isFullRbf = {};
+        const isRbf = {};
+        const isAccelerated = {};
         this.numMissing = 0;
         this.numUnexpected = 0;
 
         if (blockAudit?.template) {
           for (const tx of blockAudit.template) {
             inTemplate[tx.txid] = true;
+            if (tx.acc) {
+              isAccelerated[tx.txid] = true;
+            }
           }
           for (const tx of transactions) {
             inBlock[tx.txid] = true;
@@ -363,7 +370,10 @@ export class BlockComponent implements OnInit, OnDestroy {
             isSigop[txid] = true;
           }
           for (const txid of blockAudit.fullrbfTxs || []) {
-            isFullRbf[txid] = true;
+            isRbf[txid] = true;
+          }
+          for (const txid of blockAudit.acceleratedTxs || []) {
+            isAccelerated[txid] = true;
           }
           // set transaction statuses
           for (const tx of blockAudit.template) {
@@ -381,14 +391,17 @@ export class BlockComponent implements OnInit, OnDestroy {
                 }
               } else if (isSigop[tx.txid]) {
                 tx.status = 'sigop';
-              } else if (isFullRbf[tx.txid]) {
-                tx.status = 'fullrbf';
+              } else if (isRbf[tx.txid]) {
+                tx.status = 'rbf';
               } else {
                 tx.status = 'missing';
               }
               isMissing[tx.txid] = true;
               this.numMissing++;
             }
+            if (isAccelerated[tx.txid]) {
+              tx.status = 'accelerated';
+            }
           }
           for (const [index, tx] of transactions.entries()) {
             tx.context = 'actual';
@@ -398,13 +411,16 @@ export class BlockComponent implements OnInit, OnDestroy {
               tx.status = 'added';
             } else if (inTemplate[tx.txid]) {
               tx.status = 'found';
-            } else if (isFullRbf[tx.txid]) {
-              tx.status = 'fullrbf';
+            } else if (isRbf[tx.txid]) {
+              tx.status = 'rbf';
             } else {
               tx.status = 'selected';
               isSelected[tx.txid] = true;
               this.numUnexpected++;
             }
+            if (isAccelerated[tx.txid]) {
+              tx.status = 'accelerated';
+            }
           }
           for (const tx of transactions) {
             inBlock[tx.txid] = true;
@@ -676,7 +692,6 @@ export class BlockComponent implements OnInit, OnDestroy {
         }
         break;
       case 'signet':
-      case 'regtest':
         if (blockHeight < this.stateService.env.SIGNET_BLOCK_AUDIT_START_HEIGHT) {
           return false;
         }

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts

@@ -68,7 +68,6 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     'liquidtestnet': ['#494a4a', '#272e46'],
     testnet: ['#1d486f', '#183550'],
     signet: ['#6f1d5d', '#471850'],
-    regtest: ['#9339f4', '#105fb0'],
   };
 
   constructor(
@@ -87,7 +86,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
   ngOnInit() {
     this.dynamicBlocksAmount = Math.min(8, this.stateService.env.KEEP_BLOCKS_AMOUNT);
 
-    if (['', 'testnet', 'signet', 'regtest'].includes(this.stateService.network)) {
+    if (['', 'testnet', 'signet'].includes(this.stateService.network)) {
       this.enabledMiningInfoIfNeeded(this.location.path());
       this.location.onUrlChange((url) => this.enabledMiningInfoIfNeeded(url));
     }

frontend/src/app/components/blocks-list/blocks-list.component.ts

@@ -68,7 +68,7 @@ export class BlocksList implements OnInit {
                   for (const block of blocks) {
                     // @ts-ignore: Need to add an extra field for the template
                     block.extras.pool.logo = `/resources/mining-pools/` +
-                      block.extras.pool.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg';
+                      block.extras.pool.slug + '.svg';
                   }
                 }
                 if (this.widget) {
@@ -102,7 +102,7 @@ export class BlocksList implements OnInit {
             if (this.stateService.env.MINING_DASHBOARD) {
               // @ts-ignore: Need to add an extra field for the template
               blocks[1][0].extras.pool.logo = `/resources/mining-pools/` +
-                blocks[1][0].extras.pool.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg';
+                blocks[1][0].extras.pool.slug + '.svg';
             }
             acc.unshift(blocks[1][0]);
             acc = acc.slice(0, this.widget ? 6 : 15);

frontend/src/app/components/clock/clock.component.ts

@@ -38,7 +38,6 @@ export class ClockComponent implements OnInit {
     'liquidtestnet': ['#494a4a', '#272e46'],
     testnet: ['#1d486f', '#183550'],
     signet: ['#6f1d5d', '#471850'],
-    regtest: ['#9339f4', '#105fb0'],
   };
 
   constructor(

frontend/src/app/components/difficulty/difficulty-tooltip.component.html

@@ -4,38 +4,56 @@
   class="difficulty-tooltip"
   [style.visibility]="status ? 'visible' : 'hidden'"
   [style.left]="tooltipPosition.x + 'px'"
-  [style.top]="tooltipPosition.y + 'px'"
+  [style.top]="tooltipPosition.y + (isMobile ? -60 : 0) + 'px'"
 >
-<ng-container [ngSwitch]="status">
+<ng-container *ngIf="!isMobile" [ngSwitch]="status">
   <ng-container *ngSwitchCase="'mined'">
     <ng-container *ngIf="isAhead">
-      <ng-container *ngTemplateOutlet="expected === 1 ? blocksSingular : blocksPlural; context: {$implicit: expected }"></ng-container>
-      <ng-template #blocksPlural let-i i18n="difficulty-box.expected-blocks">{{ i }} blocks expected</ng-template>
-      <ng-template #blocksSingular let-i i18n="difficulty-box.expected-block">{{ i }} block expected</ng-template>
+      <ng-container *ngTemplateOutlet="expected === 1 ? expectedMinedBlocksSingular : expectedMinedBlocksPlural; context: {$implicit: expected }"></ng-container>
     </ng-container>
     <ng-container *ngIf="!isAhead">
-      <ng-container *ngTemplateOutlet="mined === 1 ? blocksSingular : blocksPlural; context: {$implicit: mined }"></ng-container>
-      <ng-template #blocksPlural let-i i18n="difficulty-box.mined-blocks">{{ i }} blocks mined</ng-template>
-      <ng-template #blocksSingular let-i i18n="difficulty-box.mined-block">{{ i }} block mined</ng-template>
+      <ng-container *ngTemplateOutlet="mined === 1 ? minedBlocksSingular : minedBlocksPlural; context: {$implicit: mined }"></ng-container>
     </ng-container>
   </ng-container>
   <ng-container *ngSwitchCase="'remaining'">
-    <ng-container *ngTemplateOutlet="remaining === 1 ? blocksSingular : blocksPlural; context: {$implicit: remaining }"></ng-container>
-    <ng-template #blocksPlural let-i i18n="difficulty-box.remaining-blocks">{{ i }} blocks remaining</ng-template>
-    <ng-template #blocksSingular let-i i18n="difficulty-box.remaining-block">{{ i }} block remaining</ng-template>
+    <ng-container *ngTemplateOutlet="remaining === 1 ? remainingBlocksSingular : remainingBlocksPlural; context: {$implicit: remaining }"></ng-container>
   </ng-container>
   <ng-container *ngSwitchCase="'ahead'">
-    <ng-container *ngTemplateOutlet="ahead === 1 ? blocksSingular : blocksPlural; context: {$implicit: ahead }"></ng-container>
-    <ng-template #blocksPlural let-i i18n="difficulty-box.blocks-ahead">{{ i }} blocks ahead</ng-template>
-    <ng-template #blocksSingular let-i i18n="difficulty-box.block-ahead">{{ i }} block ahead</ng-template>
+    <ng-container *ngTemplateOutlet="ahead === 1 ? aheadBlocksSingular : aheadBlocksPlural; context: {$implicit: ahead }"></ng-container>
   </ng-container>
   <ng-container *ngSwitchCase="'behind'">
-    <ng-container *ngTemplateOutlet="behind === 1 ? blocksSingular : blocksPlural; context: {$implicit: behind }"></ng-container>
-    <ng-template #blocksPlural let-i i18n="difficulty-box.blocks-behind">{{ i }} blocks behind</ng-template>
-    <ng-template #blocksSingular let-i i18n="difficulty-box.block-behind">{{ i }} block behind</ng-template>
+    <ng-container *ngTemplateOutlet="behind === 1 ? behindBlocksSingular : behindBlocksPlural; context: {$implicit: behind }"></ng-container>
   </ng-container>
   <ng-container *ngSwitchCase="'next'">
     <span class="next-block" i18n="@@bdf0e930eb22431140a2eaeacd809cc5f8ebd38c">Next Block</span>
   </ng-container>
 </ng-container>
-</div>
+<ng-container *ngIf="isMobile">
+  <ng-container *ngIf="isAhead">
+    <ng-container *ngTemplateOutlet="expected === 1 ? minedBlocksSingular : minedBlocksPlural; context: {$implicit: expected }"></ng-container>
+  </ng-container>
+  <ng-container *ngIf="!isAhead">
+    <ng-container *ngTemplateOutlet="mined === 1 ? minedBlocksSingular : minedBlocksPlural; context: {$implicit: mined }"></ng-container>
+  </ng-container>
+  <br>
+  <ng-container *ngTemplateOutlet="remaining === 1 ? remainingBlocksSingular : remainingBlocksPlural; context: {$implicit: remaining }"></ng-container>
+  <br>
+  <ng-container *ngIf="ahead > 0">
+    <ng-container *ngTemplateOutlet="ahead === 1 ? aheadBlocksSingular : aheadBlocksPlural; context: {$implicit: ahead }"></ng-container>
+  </ng-container>
+  <ng-container *ngIf="behind > 0">
+    <ng-container *ngTemplateOutlet="behind === 1 ? behindBlocksSingular : behindBlocksPlural; context: {$implicit: behind }"></ng-container>
+  </ng-container>
+</ng-container>
+</div>
+
+<ng-template #expectedMinedBlocksPlural let-i i18n="difficulty-box.expected-blocks">{{ i }} blocks expected</ng-template>
+<ng-template #expectedMinedBlocksSingular let-i i18n="difficulty-box.expected-block">{{ i }} block expected</ng-template>
+<ng-template #minedBlocksPlural let-i i18n="difficulty-box.mined-blocks">{{ i }} blocks mined</ng-template>
+<ng-template #minedBlocksSingular let-i i18n="difficulty-box.mined-block">{{ i }} block mined</ng-template>
+<ng-template #remainingBlocksPlural let-i i18n="difficulty-box.remaining-blocks">{{ i }} blocks remaining</ng-template>
+<ng-template #remainingBlocksSingular let-i i18n="difficulty-box.remaining-block">{{ i }} block remaining</ng-template>
+<ng-template #aheadBlocksPlural let-i i18n="difficulty-box.blocks-ahead">{{ i }} blocks ahead</ng-template>
+<ng-template #aheadBlocksSingular let-i i18n="difficulty-box.block-ahead">{{ i }} block ahead</ng-template>
+<ng-template #behindBlocksPlural let-i i18n="difficulty-box.blocks-behind">{{ i }} blocks behind</ng-template>
+<ng-template #behindBlocksSingular let-i i18n="difficulty-box.block-behind">{{ i }} block behind</ng-template>