Merge pull request #2585 from mempool/simon/search-bar-placeholder-update

Updating search bar placeholder

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: ['https://mempool.space/sponsor'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/ci.yml

@@ -42,8 +42,10 @@ jobs:
         run: npm run lint
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
-      #  - name: Test
-      #    run: npm run test
+      - name: Unit Tests
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run test
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Build
         run: npm run build

.github/workflows/cypress.yml

@@ -6,86 +6,53 @@ on:
 jobs:
   cypress:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
     strategy:
       fail-fast: false
       matrix:
-        containers: [1, 2, 3, 4, 5]
-        os: ["ubuntu-latest"]
-        browser: [chrome]
-    name: E2E tests on ${{ matrix.browser }} - ${{ matrix.os }}
+        module: ["mempool", "liquid", "bisq"]
+        include:
+          - module: "mempool"
+            spec: |
+              cypress/e2e/mainnet/*.spec.ts
+              cypress/e2e/signet/*.spec.ts
+              cypress/e2e/testnet/*.spec.ts
+          - module: "liquid"
+            spec: |
+              cypress/e2e/liquid/liquid.spec.ts
+              cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+          - module: "bisq"
+            spec: |
+              cypress/e2e/bisq/bisq.spec.ts
+          
+    name: E2E tests for ${{ matrix.module }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          path: ${{ matrix.module }}
+          
       - name: Setup node
         uses: actions/setup-node@v2
         with:
           node-version: 16.15.0
           cache: 'npm'
-          cache-dependency-path: frontend/package-lock.json
-      - name: ${{ matrix.browser }} browser tests (Mempool)
-        uses: cypress-io/github-action@v4
-        with:
-          tag: ${{ github.event_name }}
-          working-directory: frontend
-          build: npm run config:defaults:mempool
-          start: npm run start:local-staging
-          wait-on: 'http://localhost:4200'
-          wait-on-timeout: 120
-          record: true
-          parallel: true
-          spec: |
-            cypress/e2e/mainnet/*.spec.ts
-            cypress/e2e/signet/*.spec.ts
-            cypress/e2e/testnet/*.spec.ts
-          group: Tests on ${{ matrix.browser }} (Mempool)
-          browser: ${{ matrix.browser }}
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
-        env:
-          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+          cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 
-      - name: ${{ matrix.browser }} browser tests (Liquid)
+      - name: Chrome browser tests (${{ matrix.module }})
         uses: cypress-io/github-action@v4
-        if: always()
         with:
           tag: ${{ github.event_name }}
-          working-directory: frontend
-          build: npm run config:defaults:liquid
+          working-directory: ${{ matrix.module }}/frontend
+          build: npm run config:defaults:${{ matrix.module }}
           start: npm run start:local-staging
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
-          spec: |
-            cypress/e2e/liquid/liquid.spec.ts
-            cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
-          group: Tests on ${{ matrix.browser }} (Liquid)
-          browser: ${{ matrix.browser }}
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
-        env:
-          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
-
-      - name: ${{ matrix.browser }} browser tests (Bisq)
-        uses: cypress-io/github-action@v4
-        if: always()
-        with:
-          tag: ${{ github.event_name }}
-          working-directory: frontend
-          build: npm run config:defaults:bisq
-          start: npm run start:local-staging
-          wait-on: 'http://localhost:4200'
-          wait-on-timeout: 120
-          record: true
-          parallel: true
-          spec: cypress/e2e/bisq/bisq.spec.ts
-          group: Tests on ${{ matrix.browser }} (Bisq)
-          browser: ${{ matrix.browser }}
+          spec: ${{ matrix.spec }}
+          group: Tests on Chrome (${{ matrix.module }})
+          browser: "chrome"
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
           COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}

.github/workflows/on-tag.yml

@@ -1,7 +1,7 @@
 name: Docker build on tag
 env:
   DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
   DOCKER_BUILDKIT: 0
   COMPOSE_DOCKER_CLI_BUILD: 0
 
@@ -21,16 +21,46 @@ jobs:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
     name: Build and push to DockerHub
     steps:
+      # Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
+      - name: Replace the current swap file
+        shell: bash
+        run: |
+          sudo swapoff /mnt/swapfile
+          sudo rm -v /mnt/swapfile
+          sudo fallocate -l 10G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+
+      - name: Show current memory and swap status
+        shell: bash
+        run: |
+          sudo free -h
+          echo
+          sudo swapon --show
+
+      - name: Mount a tmpfs over /var/lib/docker
+        shell: bash
+        run: |
+          if [ ! -d "/var/lib/docker" ]; then
+            echo "Directory '/var/lib/docker' not found"
+            exit 1
+          fi
+          sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
+          sudo systemctl restart docker
+          sudo df -h | grep docker
+
       - name: Set env variables
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Show set environment variables
         run: |
           printf "      TAG: %s\n"  "$TAG"
-      
+
       - name: Add SHORT_SHA env property with commit short sha
         run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
 

backend/.eslintrc

@@ -15,10 +15,11 @@
     "@typescript-eslint/ban-types": 1,
     "@typescript-eslint/no-empty-function": 1,
     "@typescript-eslint/no-explicit-any": 1,
-    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-inferrable-types": 0,
     "@typescript-eslint/no-namespace": 1,
     "@typescript-eslint/no-this-alias": 1,
     "@typescript-eslint/no-var-requires": 1,
+    "@typescript-eslint/explicit-function-return-type": 1,
     "no-console": 1,
     "no-constant-condition": 1,
     "no-dupe-else-if": 1,
@@ -28,6 +29,9 @@
     "no-useless-catch": 1,
     "no-var": 1,
     "prefer-const": 1,
-    "prefer-rest-params": 1
+    "prefer-rest-params": 1,
+    "quotes": [1, "single", { "allowTemplateLiterals": true }],
+    "semi": 1,
+    "eqeqeq": 1
   }
 }

backend/.gitignore

@@ -1,9 +1,10 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # production config and external assets
-*.json
+!mempool-config.template.json
 !mempool-config.sample.json
-
+mempool-config.json
+pools.json
 icons.json
 
 # compiled output

backend/README.md

@@ -110,6 +110,11 @@ Run the Mempool backend:
 
 ```
 npm run start
+
+```
+You can also set env var `MEMPOOL_CONFIG_FILE` to specify a custom config file location:
+```
+MEMPOOL_CONFIG_FILE=/path/to/mempool-config.json npm run start
 ```
 
 When it's running, you should see output like this:

backend/jest.config.ts

@@ -0,0 +1,20 @@
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  verbose: true,
+  automock: false,
+  collectCoverage: true,
+  collectCoverageFrom: ["./src/**/**.ts"],
+  coverageProvider: "babel",
+  coverageThreshold: {
+    global: {
+      lines: 1
+    }
+  },
+  setupFiles: [
+    "./testSetup.ts",
+  ],
+}
+export default config;

backend/mempool-config.sample.json

@@ -21,7 +21,9 @@
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
-    "AUTOMATIC_BLOCK_REINDEXING": false
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -66,7 +68,8 @@
   "MAXMIND": {
     "ENABLED": false,
     "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
   },
   "BISQ": {
     "ENABLED": false,
@@ -74,12 +77,18 @@
   },
   "LIGHTNING": {
     "ENABLED": false,
-    "BACKEND": "lnd"
+    "BACKEND": "lnd",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30
   },
   "LND": {
     "TLS_CERT_PATH": "tls.cert",
-    "MACAROON_PATH": "admin.macaroon",
-    "SOCKET": "localhost:10009"
+    "MACAROON_PATH": "readonly.macaroon",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "lightning-rpc"
   },
   "SOCKS5PROXY": {
     "ENABLED": false,

backend/package.json

@@ -16,28 +16,31 @@
     "mempool",
     "blockchain",
     "explorer",
-    "liquid"
+    "liquid",
+    "lightning"
   ],
   "main": "index.ts",
   "scripts": {
-    "tsc": "./node_modules/typescript/bin/tsc",
-    "build": "npm run tsc",
+    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
+    "build": "npm run tsc && npm run create-resources",
+    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
+    "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
+    "@babel/core": "^7.18.6",
     "@mempool/electrum-client": "^1.1.7",
     "@types/node": "^16.11.41",
     "axios": "~0.27.2",
-    "bitcoinjs-lib": "6.0.1",
-    "bolt07": "^1.8.1",
+    "bitcoinjs-lib": "6.0.2",
     "crypto-js": "^4.0.0",
     "express": "^4.18.0",
-    "lightning": "^5.16.3",
     "maxmind": "^4.3.6",
     "mysql2": "2.3.3",
     "node-worker-threads-pool": "^1.5.1",
@@ -46,14 +49,20 @@
     "ws": "~8.8.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.18.6",
+    "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.13",
+    "@types/jest": "^28.1.4",
     "@types/ws": "~8.5.3",
     "@typescript-eslint/eslint-plugin": "^5.30.5",
     "@typescript-eslint/parser": "^5.30.5",
     "eslint": "^8.19.0",
     "eslint-config-prettier": "^8.5.0",
-    "prettier": "^2.7.1"
+    "jest": "^28.1.2",
+    "prettier": "^2.7.1",
+    "ts-jest": "^28.0.5",
+    "ts-node": "^10.8.2"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -0,0 +1,108 @@
+{
+  "MEMPOOL": {
+    "NETWORK": "__MEMPOOL_NETWORK__",
+    "BACKEND": "__MEMPOOL_BACKEND__",
+    "BLOCKS_SUMMARIES_INDEXING": true,
+    "HTTP_PORT": 1,
+    "SPAWN_CLUSTER_PROCS": 2,
+    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
+    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "POLL_RATE_MS": 3,
+    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CLEAR_PROTECTION_MINUTES": 4,
+    "RECOMMENDED_FEE_PERCENTILE": 5,
+    "BLOCK_WEIGHT_UNITS": 6,
+    "INITIAL_BLOCKS_AMOUNT": 7,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "PRICE_FEED_UPDATE_INTERVAL": 9,
+    "USE_SECOND_NODE_FOR_MINFEE": 10,
+    "EXTERNAL_ASSETS": 11,
+    "EXTERNAL_MAX_RETRY": 12,
+    "EXTERNAL_RETRY_INTERVAL": 13,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
+    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
+    "INDEXING_BLOCKS_AMOUNT": 14,
+    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__"
+  },
+  "CORE_RPC": {
+    "HOST": "__CORE_RPC_HOST__",
+    "PORT": 15,
+    "USERNAME": "__CORE_RPC_USERNAME__",
+    "PASSWORD": "__CORE_RPC_PASSWORD__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRUM_HOST__",
+    "PORT": 16,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "__SECOND_CORE_RPC_HOST__",
+    "PORT": 17,
+    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+  },
+  "DATABASE": {
+    "ENABLED": false,
+    "HOST": "__DATABASE_HOST__",
+    "SOCKET": "__DATABASE_SOCKET__",
+    "PORT": 18,
+    "DATABASE": "__DATABASE_DATABASE__",
+    "USERNAME": "__DATABASE_USERNAME__",
+    "PASSWORD": "__DATABASE_PASSWORD__"
+  },
+  "SYSLOG": {
+    "ENABLED": false,
+    "HOST": "__SYSLOG_HOST__",
+    "PORT": 19,
+    "MIN_PRIORITY": "__SYSLOG_MIN_PRIORITY__",
+    "FACILITY": "__SYSLOG_FACILITY__"
+  },
+  "STATISTICS": {
+    "ENABLED": false,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 20
+  },
+  "BISQ": {
+    "ENABLED": true,
+    "DATA_PATH": "__BISQ_DATA_PATH__"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": true,
+    "USE_ONION": true,
+    "HOST": "__SOCKS5PROXY_HOST__",
+    "PORT": "__SOCKS5PROXY_PORT__",
+    "USERNAME": "__SOCKS5PROXY_USERNAME__",
+    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
+    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
+  },
+  "LIGHTNING": {
+    "ENABLED": "__LIGHTNING_ENABLED__",
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30
+  },
+  "LND": {
+    "TLS_CERT_PATH": "",
+    "MACAROON_PATH": "",
+    "REST_API_URL": "https://localhost:8080"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  }
+}

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -0,0 +1,62 @@
+import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+
+describe('Mempool Difficulty Adjustment', () => {
+  test('should calculate Difficulty Adjustments properly', () => {
+    const dt = (dtString) => {
+      return Math.floor(new Date(dtString).getTime() / 1000);
+    };
+
+    const vectors = [
+      [ // Vector 1
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // If not testnet, not used
+        ],
+        { // Expected Result
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: 0,
+        },
+      ],
+      [ // Vector 2 (testnet)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'testnet',                      // Network
+          dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result is same other than timeOffset
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
+          // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
+        },
+      ],
+    ] as [[number, number, number, number, string, number], DifficultyAdjustment][];
+
+    for (const vector of vectors) {
+      const result = calcDifficultyAdjustment(...vector[0]);
+      // previousRetarget is passed through untouched
+      expect(result.previousRetarget).toStrictEqual(vector[0][3]);
+      expect(result).toStrictEqual(vector[1]);
+    }
+  });
+});

backend/src/__tests__/config.test.ts

@@ -0,0 +1,139 @@
+import * as fs from 'fs';
+
+describe('Mempool Backend Config', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  test('should return defaults when no file is present', () => {
+    jest.isolateModules(() => {
+      jest.mock('../../mempool-config.json', () => ({}), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual({
+        NETWORK: 'mainnet',
+        BACKEND: 'none',
+        BLOCKS_SUMMARIES_INDEXING: false,
+        HTTP_PORT: 8999,
+        SPAWN_CLUSTER_PROCS: 0,
+        API_URL_PREFIX: '/api/v1/',
+        AUTOMATIC_BLOCK_REINDEXING: false,
+        POLL_RATE_MS: 2000,
+        CACHE_DIR: './cache',
+        CLEAR_PROTECTION_MINUTES: 20,
+        RECOMMENDED_FEE_PERCENTILE: 50,
+        BLOCK_WEIGHT_UNITS: 4000000,
+        INITIAL_BLOCKS_AMOUNT: 8,
+        MEMPOOL_BLOCKS_AMOUNT: 8,
+        INDEXING_BLOCKS_AMOUNT: 11000,
+        PRICE_FEED_UPDATE_INTERVAL: 600,
+        USE_SECOND_NODE_FOR_MINFEE: false,
+        EXTERNAL_ASSETS: [],
+        EXTERNAL_MAX_RETRY: 1,
+        EXTERNAL_RETRY_INTERVAL: 0,
+        USER_AGENT: 'mempool',
+        STDOUT_LOG_MIN_PRIORITY: 'debug',
+        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'
+      });
+
+      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
+
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+
+      expect(config.CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.DATABASE).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        SOCKET: '',
+        PORT: 3306,
+        DATABASE: 'mempool',
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SYSLOG).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        PORT: 514,
+        MIN_PRIORITY: 'info',
+        FACILITY: 'local7'
+      });
+
+      expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
+
+      expect(config.BISQ).toStrictEqual({ ENABLED: false, DATA_PATH: '/bisq/statsnode-data/btc_mainnet/db' });
+
+      expect(config.SOCKS5PROXY).toStrictEqual({
+        ENABLED: false,
+        USE_ONION: true,
+        HOST: '127.0.0.1',
+        PORT: 9050,
+        USERNAME: '',
+        PASSWORD: ''
+      });
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual({
+        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
+      });
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
+        MEMPOOL_API: 'https://mempool.space/api/v1',
+        MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+        LIQUID_API: 'https://liquid.network/api/v1',
+        LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+        BISQ_URL: 'https://bisq.markets/api',
+        BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+      });
+    });
+  });
+
+  test('should override the default values with the passed values', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      jest.mock('../../mempool-config.json', () => (fixture), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual(fixture.MEMPOOL);
+
+      expect(config.ELECTRUM).toStrictEqual(fixture.ELECTRUM);
+
+      expect(config.ESPLORA).toStrictEqual(fixture.ESPLORA);
+
+      expect(config.CORE_RPC).toStrictEqual(fixture.CORE_RPC);
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual(fixture.SECOND_CORE_RPC);
+
+      expect(config.DATABASE).toStrictEqual(fixture.DATABASE);
+
+      expect(config.SYSLOG).toStrictEqual(fixture.SYSLOG);
+
+      expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
+
+      expect(config.BISQ).toStrictEqual(fixture.BISQ);
+
+      expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+    });
+  });
+});

backend/src/api/backend-info.ts

@@ -1,60 +1,37 @@
-import * as fs from 'fs';
-import * as os from 'os';
-import logger from '../logger';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
-const { spawnSync } = require('child_process');
 
 class BackendInfo {
-  private gitCommitHash = '';
-  private hostname = '';
-  private version = '';
+  private backendInfo: IBackendInfo;
 
   constructor() {
-    this.setLatestCommitHash();
-    this.setVersion();
-    this.hostname = os.hostname();
-  }
-
-  public getBackendInfo(): IBackendInfo {
-    return {
-      hostname: this.hostname,
-      gitCommit: this.gitCommitHash,
-      version: this.version,
+    // This file is created by ./fetch-version.ts during building
+    const versionFile = path.join(__dirname, 'version.json')
+    var versionInfo;
+    if (fs.existsSync(versionFile)) {
+      versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
+    } else {
+      // Use dummy values if `versionFile` doesn't exist (e.g., during testing)
+      versionInfo = {
+        version: '?',
+        gitCommit: '?'
+      };
+    }
+    this.backendInfo = {
+      hostname: os.hostname(),
+      version: versionInfo.version,
+      gitCommit: versionInfo.gitCommit
     };
   }
 
+  public getBackendInfo(): IBackendInfo {
+    return this.backendInfo;
+  }
+
   public getShortCommitHash() {
-    return this.gitCommitHash.slice(0, 7);
-  }
-
-  private setLatestCommitHash(): void {
-    //TODO: share this logic with `generate-config.js`
-    if (process.env.DOCKER_COMMIT_HASH) {
-      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
-    } else {
-      try {
-        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
-        if (!gitRevParse.error) {
-          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
-          this.gitCommitHash = output ? output : '?';
-        } else if (gitRevParse.error.code === 'ENOENT') {
-          console.log('git not found, cannot parse git hash');
-          this.gitCommitHash = '?';
-        }
-      } catch (e: any) {
-        console.log('Could not load git commit info: ' + e.message);
-        this.gitCommitHash = '?';
-      }
-    }
-  }
-
-  private setVersion(): void {
-    try {
-      const packageJson = fs.readFileSync('package.json').toString();
-      this.version = JSON.parse(packageJson).version;
-    } catch (e) {
-      throw new Error(e instanceof Error ? e.message : 'Error');
-    }
+    return this.backendInfo.gitCommit.slice(0, 7);
   }
 }
 

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -9,6 +9,7 @@ export interface AbstractBitcoinApi {
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
+  $getRawBlock(hash: string): Promise<string>;
   $getAddress(address: string): Promise<IEsploraApi.Address>;
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];

backend/src/api/bitcoin/bitcoin-api.ts

@@ -77,7 +77,8 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   $getRawBlock(hash: string): Promise<string> {
-    return this.bitcoindClient.getBlock(hash, 0);
+    return this.bitcoindClient.getBlock(hash, 0)
+      .then((raw: string) => Buffer.from(raw, "hex"));
   }
 
   $getBlockHash(height: number): Promise<string> {

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -103,9 +103,10 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', this.getBlockTransactions)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', this.getBlockTransactions)
-          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
           .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
@@ -470,6 +471,16 @@ class BitcoinRoutes {
     }
   }
 
+  private async getRawBlock(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getRawBlock(req.params.hash);
+      res.setHeader('content-type', 'application/octet-stream');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getTxIdsForBlock(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
@@ -499,7 +510,12 @@ class BitcoinRoutes {
 
   private getDifficultyChange(req: Request, res: Response) {
     try {
-      res.json(difficultyAdjustment.getDifficultyAdjustment());
+      const da = difficultyAdjustment.getDifficultyAdjustment();
+      if (da) {
+        res.json(da);
+      } else {
+        res.status(503).send(`Service Temporarily Unavailable`);
+      }
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/bitcoin/esplora-api.ts

@@ -50,6 +50,11 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
+  $getRawBlock(hash: string): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", this.axiosConfig)
+      .then((response) => response.data);
+  }
+
   $getAddress(address: string): Promise<IEsploraApi.Address> {
     throw new Error('Method getAddress not implemented.');
   }

backend/src/api/blocks.ts

@@ -22,6 +22,8 @@ import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import PricesRepository from '../repositories/PricesRepository';
+import priceUpdater from '../tasks/price-updater';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -457,6 +459,19 @@ class Blocks {
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
 
+          const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
+            await blocksRepository.$saveBlockPrices([{
+              height: blockExtended.height,
+              priceId: lastestPriceId,
+            }]);
+          } else {
+            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
+            setTimeout(() => {
+              indexer.runSingleTask('blocksPrices');
+            }, 10000);
+          }
+
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
@@ -578,7 +593,7 @@ class Blocks {
 
     // Index the response if needed
     if (Common.blocksSummariesIndexingEnabled() === true) {
-      await BlocksSummariesRepository.$saveSummary(block.height, summary, null);
+      await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary});
     }
 
     return summary.transactions;

backend/src/api/common.ts

@@ -1,5 +1,7 @@
 import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
+import { NodeSocket } from '../repositories/NodesSocketsRepository';
+import { isIP } from 'net';
 export class Common {
   static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
     '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -184,4 +186,117 @@ export class Common {
       config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
     );
   }
+
+  static setDateMidnight(date: Date): void {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+  }
+
+  static channelShortIdToIntegerId(channelId: string): string {
+    if (channelId.indexOf('x') === -1) { // Already an integer id
+      return channelId;
+    }
+    if (channelId.indexOf('/') !== -1) { // Topology import
+      channelId = channelId.slice(0, -2);
+    }
+    const s = channelId.split('x').map(part => BigInt(part));
+    return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
+  }
+
+  /** Decodes a channel id returned by lnd as uint64 to a short channel id */
+  static channelIntegerIdToShortId(id: string): string {
+    if (id.indexOf('/') !== -1) {
+      id = id.slice(0, -2);
+    }
+    
+    if (id.indexOf('x') !== -1) { // Already a short id
+      return id;
+    }
+
+    const n = BigInt(id);
+    return [
+      n >> 40n, // nth block
+      (n >> 16n) & 0xffffffn, // nth tx of the block
+      n & 0xffffn // nth output of the tx
+    ].join('x');
+  }
+
+  static utcDateToMysql(date?: number): string {
+    const d = new Date((date || 0) * 1000);
+    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
+  }
+
+  static findSocketNetwork(addr: string): {network: string | null, url: string} {
+    let network: string | null = null;
+    let url = addr.split('://')[1];
+
+    if (!url) {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    if (addr.indexOf('onion') !== -1) {
+      if (url.split('.')[0].length >= 56) {
+        network = 'torv3';
+      } else {
+        network = 'torv2';
+      }
+    } else if (addr.indexOf('i2p') !== -1) {
+      network = 'i2p';
+    } else if (addr.indexOf('ipv4') !== -1) {
+      const ipv = isIP(url.split(':')[0]);
+      if (ipv === 4) {
+        network = 'ipv4';
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else if (addr.indexOf('ipv6') !== -1) {
+      url = url.split('[')[1].split(']')[0];
+      const ipv = isIP(url);
+      if (ipv === 6) {
+        const parts = addr.split(':');
+        network = 'ipv6';
+        url = `[${url}]:${parts[parts.length - 1]}`;
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    return {
+      network: network,
+      url: url,
+    };
+  }
+
+  static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      return {
+        publicKey: publicKey,
+        network: socket.network,
+        addr: socket.addr,
+      };
+    } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
+      const formatted = this.findSocketNetwork(socket.addr);
+      return {
+        publicKey: publicKey,
+        network: formatted.network,
+        addr: formatted.url,
+      };
+    }
+  }
 }

backend/src/api/database-migration.ts

@@ -4,13 +4,13 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 32;
+  private static currentVersion = 40;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
 
-  private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`;
-  private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`;
+  private blocksTruncatedMessage = `'blocks' table has been truncated.`;
+  private hashratesTruncatedMessage = `'hashrates' table has been truncated.`;
 
   /**
    * Avoid printing multiple time the same message
@@ -248,7 +248,6 @@ class DatabaseMigration {
     }
 
     if (databaseSchemaVersion < 25 && isBitcoin === true) {
-      await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
       await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
       await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
       await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
@@ -256,7 +255,9 @@ class DatabaseMigration {
     }
 
     if (databaseSchemaVersion < 26 && isBitcoin === true) {
-      this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`);
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated.`);
+      }
       await this.$executeQuery(`TRUNCATE lightning_stats`);
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
@@ -273,6 +274,9 @@ class DatabaseMigration {
     }
     
     if (databaseSchemaVersion < 28 && isBitcoin === true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
+      }
       await this.$executeQuery(`TRUNCATE lightning_stats`);
       await this.$executeQuery(`TRUNCATE node_stats`);
       await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
@@ -302,6 +306,48 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 32 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
     }
+
+    if (databaseSchemaVersion < 33 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
+    }
+
+    if (databaseSchemaVersion < 34 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
+    }
+
+    if (databaseSchemaVersion < 35 && isBitcoin == true) {
+      await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
+    }
+
+    if (databaseSchemaVersion < 36 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
+    }
+
+    if (databaseSchemaVersion < 37 && isBitcoin == true) {
+      await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+    }
+
+    if (databaseSchemaVersion < 38 && isBitcoin == true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
+      }
+      await this.$executeQuery(`TRUNCATE lightning_stats`);
+      await this.$executeQuery(`TRUNCATE node_stats`);
+      await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
+      await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
+    }
+
+    if (databaseSchemaVersion < 39 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
+      await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
+    }
+
+    if (databaseSchemaVersion < 40 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
+    }
   }
 
   /**
@@ -715,7 +761,7 @@ class DatabaseMigration {
       names text DEFAULT NULL,
       UNIQUE KEY id (id,type),
       KEY id_2 (id)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
   private getCreateBlocksPricesTableQuery(): string {
@@ -727,6 +773,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodesSocketsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_sockets (
+      public_key varchar(66) NOT NULL,
+      socket varchar(100) NOT NULL,
+      type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL,
+      UNIQUE KEY public_key_socket (public_key, socket),
+      INDEX (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/difficulty-adjustment.ts

@@ -2,65 +2,100 @@ import config from '../config';
 import { IDifficultyAdjustment } from '../mempool.interfaces';
 import blocks from './blocks';
 
-class DifficultyAdjustmentApi {
-  constructor() { }
+export interface DifficultyAdjustment {
+  progressPercent: number;       // Percent: 0 to 100
+  difficultyChange: number;      // Percent: -75 to 300
+  estimatedRetargetDate: number; // Unix time in ms
+  remainingBlocks: number;       // Block count
+  remainingTime: number;         // Duration of time in ms
+  previousRetarget: number;      // Percent: -75 to 300
+  nextRetargetHeight: number;    // Block Height
+  timeAvg: number;               // Duration of time in ms
+  timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
+}
 
-  public getDifficultyAdjustment(): IDifficultyAdjustment {
+export function calcDifficultyAdjustment(
+  DATime: number,
+  nowSeconds: number,
+  blockHeight: number,
+  previousRetarget: number,
+  network: string,
+  latestBlockTimestamp: number,
+): DifficultyAdjustment {
+  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
+  const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
+  const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
+  const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
+
+  const diffSeconds = nowSeconds - DATime;
+  const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
+  const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
+  const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
+  const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
+
+  let difficultyChange = 0;
+  let timeAvgSecs = BLOCK_SECONDS_TARGET;
+  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+    timeAvgSecs = diffSeconds / blocksInEpoch;
+    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+    // Max increase is x4 (+300%)
+    if (difficultyChange > 300) {
+      difficultyChange = 300;
+    }
+    // Max decrease is /4 (-75%)
+    if (difficultyChange < -75) {
+      difficultyChange = -75;
+    }
+  }
+
+  // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+  // therefore the time between blocks will always be below 20 minutes (1200s).
+  let timeOffset = 0;
+  if (network === 'testnet') {
+    if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
+    }
+
+    const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
+    if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
+    }
+  }
+
+  const timeAvg = Math.floor(timeAvgSecs * 1000);
+  const remainingTime = remainingBlocks * timeAvg;
+  const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
+
+  return {
+    progressPercent,
+    difficultyChange,
+    estimatedRetargetDate,
+    remainingBlocks,
+    remainingTime,
+    previousRetarget,
+    nextRetargetHeight,
+    timeAvg,
+    timeOffset,
+  };
+}
+
+class DifficultyAdjustmentApi {
+  public getDifficultyAdjustment(): IDifficultyAdjustment | null {
     const DATime = blocks.getLastDifficultyAdjustmentTime();
     const previousRetarget = blocks.getPreviousDifficultyRetarget();
     const blockHeight = blocks.getCurrentBlockHeight();
     const blocksCache = blocks.getBlocks();
     const latestBlock = blocksCache[blocksCache.length - 1];
-
-    const now = new Date().getTime() / 1000;
-    const diff = now - DATime;
-    const blocksInEpoch = blockHeight % 2016;
-    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
-    const remainingBlocks = 2016 - blocksInEpoch;
-    const nextRetargetHeight = blockHeight + remainingBlocks;
-
-    let difficultyChange = 0;
-    if (remainingBlocks < 1870) {
-      if (blocksInEpoch > 0) {
-        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
-      }
-      if (difficultyChange > 300) {
-        difficultyChange = 300;
-      }
-      if (difficultyChange < -75) {
-        difficultyChange = -75;
-      }
+    if (!latestBlock) {
+      return null;
     }
+    const nowSeconds = Math.floor(new Date().getTime() / 1000);
 
-    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
-
-    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
-    // therefore the time between blocks will always be below 20 minutes (1200s).
-    let timeOffset = 0;
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      if (timeAvgMins > 20) {
-        timeAvgMins = 20;
-      }
-      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
-        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
-      }
-    }
-
-    const timeAvg = timeAvgMins * 60 * 1000 ;
-    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
-    const estimatedRetargetDate = remainingTime + now;
-
-    return {
-      progressPercent,
-      difficultyChange,
-      estimatedRetargetDate,
-      remainingBlocks,
-      remainingTime,
-      previousRetarget,
-      nextRetargetHeight,
-      timeAvg,
-      timeOffset,
-    };
+    return calcDifficultyAdjustment(
+      DATime, nowSeconds, blockHeight, previousRetarget,
+      config.MEMPOOL.NETWORK, latestBlock.timestamp
+    );
   }
 }
 

backend/src/api/explorer/channels.api.ts

@@ -1,5 +1,9 @@
 import logger from '../../logger';
 import DB from '../../database';
+import nodesApi from './nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { Common } from '../common';
 
 class ChannelsApi {
   public async $getAllChannels(): Promise<any[]> {
@@ -13,10 +17,71 @@ class ChannelsApi {
     }
   }
 
+  public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
+    try {
+      let select: string;
+      if (style === 'widget') {
+        select = `
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      } else {
+        select = `
+          nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      }
+
+      const params: string[] = [];
+      let query = `SELECT ${select}
+        FROM channels
+        JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
+        JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
+        WHERE channels.status = 1
+          AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
+          AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
+      `;
+
+      if (publicKey !== undefined) {
+        query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
+        params.push(publicKey);
+        params.push(publicKey);
+      } else {
+        query += ` AND channels.capacity > 1000000
+          GROUP BY nodes_1.public_key, nodes_2.public_key
+          ORDER BY channels.capacity DESC
+          LIMIT 10000
+        `;        
+      }
+
+      const [rows]: any = await DB.query(query, params);
+      return rows.map((row) => {
+        if (style === 'widget') {
+          return [
+            row.node1_longitude, row.node1_latitude,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        } else {
+          return [
+            row.node1_public_key, row.node1_alias,
+            row.node1_longitude, row.node1_latitude,
+            row.node2_public_key, row.node2_alias,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        }
+      });
+    } catch (e) {
+      logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $searchChannelsById(search: string): Promise<any[]> {
     try {
       const searchStripped = search.replace('%', '') + '%';
-      const query = `SELECT id, short_id, capacity FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
+      const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
       const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
       return rows;
     } catch (e) {
@@ -25,9 +90,14 @@ class ChannelsApi {
     }
   }
 
-  public async $getChannelsByStatus(status: number): Promise<any[]> {
+  public async $getChannelsByStatus(status: number | number[]): Promise<any[]> {
     try {
-      const query = `SELECT * FROM channels WHERE status = ?`;
+      let query: string;
+      if (Array.isArray(status)) {
+        query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`;
+      } else {
+        query = `SELECT * FROM channels WHERE status = ?`;
+      }
       const [rows]: any = await DB.query(query, [status]);
       return rows;
     } catch (e) {
@@ -60,7 +130,31 @@ class ChannelsApi {
 
   public async $getChannel(id: string): Promise<any> {
     try {
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND channels.id = ?`;
+      const query = `
+        SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
+          n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
+          channels.*,
+          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
+        LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
+        WHERE (
+          ns1.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node1_public_key
+          )
+          AND ns2.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node2_public_key
+          )
+        )
+        AND channels.id = ?
+      `;
+
       const [rows]: any = await DB.query(query, [id]);
       if (rows[0]) {
         return this.convertChannel(rows[0]);
@@ -136,9 +230,14 @@ class ChannelsApi {
 
   public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
     try {
-      transactionIds = transactionIds.map((id) => '\'' + id + '\'');
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.transaction_id IN (${transactionIds.join(', ')}) OR channels.closing_transaction_id IN (${transactionIds.join(', ')})`;
-      const [rows]: any = await DB.query(query);
+      const query = `
+        SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
+      `;
+      const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
       const channels = rows.map((row) => this.convertChannel(row));
       return channels;
     } catch (e) {
@@ -149,15 +248,95 @@ class ChannelsApi {
 
   public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
     try {
-      // Default active and inactive channels
-      let statusQuery = '< 2';
-      // Closed channels only
-      if (status === 'closed') {
-        statusQuery = '= 2';
+      let channelStatusFilter;
+      if (status === 'open') {
+        channelStatusFilter = '< 2';
+      } else if (status === 'active') {
+        channelStatusFilter = '= 1';
+      } else if (status === 'closed') {
+        channelStatusFilter = '= 2';
+      } else {
+        throw new Error('getChannelsForNode: Invalid status requested');
       }
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`;
-      const [rows]: any = await DB.query(query, [public_key, public_key, index, length]);
-      const channels = rows.map((row) => this.convertChannel(row));
+
+      // Channels originating from node
+      let query = `
+        SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
+          channels.status, channels.node1_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
+        WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsFromNode]: any = await DB.query(query, [public_key]);
+
+      // Channels incoming to node
+      query = `
+        SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
+          channels.status, channels.node2_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
+        WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsToNode]: any = await DB.query(query, [public_key]);
+
+      let allChannels = channelsFromNode.concat(channelsToNode);
+      allChannels.sort((a, b) => {
+        if (status === 'closed') {
+          if (!b.closing_date && !a.closing_date) {
+            return (b.updated_at ?? 0) - (a.updated_at ?? 0);
+          } else {
+            return (b.closing_date ?? 0) - (a.closing_date ?? 0);
+          }
+        } else {
+          return b.capacity - a.capacity;
+        }
+      });
+
+      if (index >= 0) {
+        allChannels = allChannels.slice(index, index + length);
+      } else if (index === -1) { // Node channels tree chart
+        allChannels = allChannels.slice(0, 1000);
+      }
+
+      const channels: any[] = []
+      for (const row of allChannels) {
+        let channel;
+        if (index >= 0) {
+          const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
+          channel = {
+            status: row.status,
+            closing_reason: row.closing_reason,
+            closing_date: row.closing_date,
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+              channels: activeChannelsStats.active_channel_count ?? 0,
+              capacity: activeChannelsStats.capacity ?? 0,
+            }
+          };
+        } else if (index === -1) {
+          channel = {
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+            }
+          };
+        }
+
+        channels.push(channel);
+      }
+
       return channels;
     } catch (e) {
       logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
@@ -173,7 +352,12 @@ class ChannelsApi {
       if (status === 'closed') {
         statusQuery = '= 2';
       }
-      const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`;
+      const query = `
+        SELECT COUNT(*) AS count
+        FROM channels
+        WHERE (node1_public_key = ? OR node2_public_key = ?)
+        AND status ${statusQuery}
+      `;
       const [rows]: any = await DB.query(query, [public_key, public_key]);
       return rows[0]['count'];
     } catch (e) {
@@ -191,6 +375,7 @@ class ChannelsApi {
       'transaction_vout': channel.transaction_vout,
       'closing_transaction_id': channel.closing_transaction_id,
       'closing_reason': channel.closing_reason,
+      'closing_date': channel.closing_date,
       'updated_at': channel.updated_at,
       'created': channel.created,
       'status': channel.status,
@@ -206,6 +391,8 @@ class ChannelsApi {
         'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
         'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
         'updated_at': channel.node1_updated_at,
+        'longitude': channel.node1_longitude,
+        'latitude': channel.node1_latitude,
       },
       'node_right': {
         'alias': channel.alias_right,
@@ -219,9 +406,157 @@ class ChannelsApi {
         'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
         'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
         'updated_at': channel.node2_updated_at,
+        'longitude': channel.node2_longitude,
+        'latitude': channel.node2_latitude,
       },
     };
   }
+
+  /**
+   * Save or update a channel present in the graph
+   */
+  public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
+    const [ txid, vout ] = channel.chan_point.split(':');
+
+    const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
+    const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
+
+    const query = `INSERT INTO channels
+      (
+        id,
+        short_id,
+        capacity,
+        transaction_id,
+        transaction_vout,
+        updated_at,
+        status,
+        node1_public_key,
+        node1_base_fee_mtokens,
+        node1_cltv_delta,
+        node1_fee_rate,
+        node1_is_disabled,
+        node1_max_htlc_mtokens,
+        node1_min_htlc_mtokens,
+        node1_updated_at,
+        node2_public_key,
+        node2_base_fee_mtokens,
+        node2_cltv_delta,
+        node2_fee_rate,
+        node2_is_disabled,
+        node2_max_htlc_mtokens,
+        node2_min_htlc_mtokens,
+        node2_updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+        capacity = ?,
+        updated_at = ?,
+        status = ${status},
+        node1_public_key = ?,
+        node1_base_fee_mtokens = ?,
+        node1_cltv_delta = ?,
+        node1_fee_rate = ?,
+        node1_is_disabled = ?,
+        node1_max_htlc_mtokens = ?,
+        node1_min_htlc_mtokens = ?,
+        node1_updated_at = ?,
+        node2_public_key = ?,
+        node2_base_fee_mtokens = ?,
+        node2_cltv_delta = ?,
+        node2_fee_rate = ?,
+        node2_is_disabled = ?,
+        node2_max_htlc_mtokens = ?,
+        node2_min_htlc_mtokens = ?,
+        node2_updated_at = ?
+      ;`;
+
+    await DB.query(query, [
+      Common.channelShortIdToIntegerId(channel.channel_id),
+      Common.channelIntegerIdToShortId(channel.channel_id),
+      channel.capacity,
+      txid,
+      vout,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update),
+      channel.capacity,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update)
+    ]);
+  }
+
+  /**
+   * Set all channels not in `graphChannelsIds` as inactive (status = 0)
+   */
+  public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
+    if (graphChannelsIds.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE id NOT IN (
+          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
+        )
+        AND status != 2
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
+    try {
+      const query = `
+        SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
+        FROM channels
+        WHERE node1_public_key = ?
+      `;
+      const [rows]: any[] = await DB.query(query, [publicKey]);
+      if (rows.length > 0) {
+        return rows[0].updated_at;
+      }
+    } catch (e) {
+      logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
+    }
+    return 0;
+  }
 }
 
 export default new ChannelsApi();

backend/src/api/explorer/channels.routes.ts

@@ -11,6 +11,8 @@ class ChannelsRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getAllChannelsGeo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo/:publicKey', this.$getAllChannelsGeo)
     ;
   }
 
@@ -30,6 +32,9 @@ class ChannelsRoutes {
         res.status(404).send('Channel not found');
         return;
       }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channel);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -42,11 +47,22 @@ class ChannelsRoutes {
         res.status(400).send('Missing parameter: public_key');
         return;
       }
+
       const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
       const status: string = typeof req.query.status === 'string' ? req.query.status : '';
-      const length = 25;
-      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status);
+
+      if (index < -1) {
+        res.status(400).send('Invalid index');
+      }
+      if (['open', 'active', 'closed'].includes(status) === false) {
+        res.status(400).send('Invalid status');
+      }
+
+      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
       const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.header('X-Total-Count', channelsCount.toString());
       res.json(channels);
     } catch (e) {
@@ -54,7 +70,7 @@ class ChannelsRoutes {
     }
   }
 
-  private async $getChannelsByTransactionIds(req: Request, res: Response) {
+  private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
     try {
       if (!Array.isArray(req.query.txId)) {
         res.status(400).send('Not an array');
@@ -67,27 +83,39 @@ class ChannelsRoutes {
         }
       }
       const channels = await channelsApi.$getChannelsByTransactionId(txIds);
-      const inputs: any[] = [];
-      const outputs: any[] = [];
+      const result: any[] = [];
       for (const txid of txIds) {
-        const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid);
-        if (foundChannelInputs) {
-          inputs.push(foundChannelInputs);
-        } else {
-          inputs.push(null);
+        const inputs: any = {};
+        const outputs: any = {};
+        // Assuming that we only have one lightning close input in each transaction. This may not be true in the future
+        const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
+        if (foundChannelsFromInput) {
+          inputs[0] = foundChannelsFromInput;
         }
-        const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid);
-        if (foundChannelOutputs) {
-          outputs.push(foundChannelOutputs);
-        } else {
-          outputs.push(null);
+        const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
+        for (const output of foundChannelsFromOutputs) {
+          outputs[output.transaction_vout] = output;
         }
+        result.push({
+          inputs,
+          outputs,
+        });
       }
 
-      res.json({
-        inputs: inputs,
-        outputs: outputs,
-      });
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getAllChannelsGeo(req: Request, res: Response) {
+    try {
+      const style: string = typeof req.query.style === 'string' ? req.query.style : '';
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channels);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/explorer/nodes.api.ts

@@ -1,43 +1,134 @@
 import logger from '../../logger';
 import DB from '../../database';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
 
 class NodesApi {
+  public async $getWorldNodes(): Promise<any> {
+    try {
+      let query = `
+        SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+        CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+        CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+        nodes.longitude, nodes.latitude,
+        geo_names_country.names as country, geo_names_iso.names as isoCode
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+        ORDER BY capacity
+      `;
+
+      const [nodes]: any[] = await DB.query(query);
+
+      for (let i = 0; i < nodes.length; ++i) {
+        nodes[i].country = JSON.parse(nodes[i].country);
+      }
+
+      query = `
+        SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
+        FROM nodes
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+      `;
+
+      const [maximums]: any[] = await DB.query(query);
+      
+      return {
+        maxLiquidity: maximums[0].maxLiquidity,
+        maxChannels: maximums[0].maxChannels,
+        nodes: nodes.map(node => [
+          node.longitude, node.latitude,
+          node.publicKey, node.alias, node.capacity, node.channels,
+          node.country, node.isoCode
+        ])
+      };
+    } catch (e) {
+      logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
   public async $getNode(public_key: string): Promise<any> {
     try {
-      const query = `
-        SELECT nodes.*, geo_names_as.names as as_organization, geo_names_city.names as city,
-        geo_names_country.names as country, geo_names_subdivision.names as subdivision,
-          (SELECT Count(*)
-          FROM channels
-          WHERE channels.status < 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_count,
-          (SELECT Sum(capacity)
-          FROM channels
-          WHERE channels.status < 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity,
-          (SELECT Avg(capacity)
-          FROM channels
-          WHERE status < 2 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg
+      // General info
+      let query = `
+        SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
+          UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
+          as_number, city_id, country_id, subdivision_id, longitude, latitude,
+          geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
+          geo_names_country.names as country, geo_names_subdivision.names as subdivision
         FROM nodes
         LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
         LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
         LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = subdivision_id
         LEFT JOIN geo_names geo_names_country on geo_names_country.id = country_id
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
         WHERE public_key = ?
       `;
-      const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key]);
-      if (rows.length > 0) {
-        rows[0].as_organization = JSON.parse(rows[0].as_organization);
-        rows[0].subdivision = JSON.parse(rows[0].subdivision);
-        rows[0].city = JSON.parse(rows[0].city);
-        rows[0].country = JSON.parse(rows[0].country);
-        return rows[0];
+      let [rows]: any[] = await DB.query(query, [public_key]);
+      if (rows.length === 0) {
+        throw new Error(`This node does not exist, or our node is not seeing it yet`);
       }
-      return null;
+
+      const node = rows[0];
+      node.as_organization = JSON.parse(node.as_organization);
+      node.subdivision = JSON.parse(node.subdivision);
+      node.city = JSON.parse(node.city);
+      node.country = JSON.parse(node.country);
+
+      // Active channels and capacity
+      const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
+      node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
+      node.capacity = activeChannelsStats.capacity ?? 0;
+
+      // Opened channels count
+      query = `
+        SELECT count(short_id) as opened_channel_count
+        FROM channels
+        WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.opened_channel_count = 0;
+      if (rows.length > 0) {
+        node.opened_channel_count = rows[0].opened_channel_count;
+      }
+
+      // Closed channels count
+      query = `
+        SELECT count(short_id) as closed_channel_count
+        FROM channels
+        WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.closed_channel_count = 0;
+      if (rows.length > 0) {
+        node.closed_channel_count = rows[0].closed_channel_count;
+      }
+
+      return node;
     } catch (e) {
-      logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
 
+  public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
+    const query = `
+      SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
+      FROM channels
+      WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+    `;
+    const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
+    if (rows.length > 0) {
+      return {
+        active_channel_count: rows[0].active_channel_count,
+        capacity: rows[0].capacity
+      };
+    } else {
+      return null;
+    }
+  }
+
   public async $getAllNodes(): Promise<any> {
     try {
       const query = `SELECT * FROM nodes`;
@@ -51,7 +142,12 @@ class NodesApi {
 
   public async $getNodeStats(public_key: string): Promise<any> {
     try {
-      const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;
+      const query = `
+        SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
+        FROM node_stats
+        WHERE public_key = ?
+        ORDER BY added DESC
+      `;
       const [rows]: any = await DB.query(query, [public_key]);
       return rows;
     } catch (e) {
@@ -60,10 +156,44 @@ class NodesApi {
     }
   }
 
-  public async $getTopCapacityNodes(): Promise<any> {
+  public async $getTopCapacityNodes(full: boolean): Promise<ITopNodesPerCapacity[]> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query);
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.capacity
+          FROM nodes
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
       return rows;
     } catch (e) {
       logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
@@ -71,10 +201,95 @@ class NodesApi {
     }
   }
 
-  public async $getTopChannelsNodes(): Promise<any> {
+  public async $getTopChannelsNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query);
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels
+          FROM nodes
+          ORDER BY channels DESC
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
+      const latestDate = rows[0].maxAdded;
+
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            node_stats.channels
+          FROM node_stats
+          JOIN nodes ON nodes.public_key = node_stats.public_key
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
+            CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM node_stats
+          RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
       return rows;
     } catch (e) {
       logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
@@ -84,15 +299,307 @@ class NodesApi {
 
   public async $searchNodeByPublicKeyOrAlias(search: string) {
     try {
-      const searchStripped = search.replace('%', '') + '%';
-      const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR nodes.alias LIKE ? GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`;
-      const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
+      const publicKeySearch = search.replace('%', '') + '%';
+      const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
+      const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
       return rows;
     } catch (e) {
       logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
+
+  public async $getNodesISPRanking() {
+    try {
+      let query = '';
+
+      // List all channels and the two linked ISP
+      query = `
+        SELECT short_id, channels.capacity,
+          channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1
+        ORDER BY short_id DESC
+      `;
+      const [channelsIsp]: any = await DB.query(query);
+
+      // Sum channels capacity and node count per ISP
+      const ispList = {};
+      for (const channel of channelsIsp) {
+        const isp1 = JSON.parse(channel.isp1);
+        const isp2 = JSON.parse(channel.isp2);
+
+        if (!ispList[isp1]) {
+          ispList[isp1] = {
+            id: channel.isp1ID.toString(),
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp1].id.indexOf(channel.isp1ID) === -1) {
+          ispList[isp1].id += ',' + channel.isp1ID.toString();
+        }
+
+        if (!ispList[isp2]) {
+          ispList[isp2] = {
+            id: channel.isp2ID.toString(),
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp2].id.indexOf(channel.isp2ID) === -1) {
+          ispList[isp2].id += ',' + channel.isp2ID.toString();
+        }
+        
+        ispList[isp1].capacity += channel.capacity;
+        ispList[isp1].channels += 1;
+        ispList[isp1].nodes[channel.node1PublicKey] = true;
+        ispList[isp2].capacity += channel.capacity;
+        ispList[isp2].channels += 1;
+        ispList[isp2].nodes[channel.node2PublicKey] = true;
+      }
+
+      const ispRanking: any[] = [];
+      for (const isp of Object.keys(ispList)) {
+        ispRanking.push([
+          ispList[isp].id,
+          isp,
+          ispList[isp].capacity,
+          ispList[isp].channels,
+          Object.keys(ispList[isp].nodes).length,
+        ]);
+      }
+
+      // Total active channels capacity
+      query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`;
+      const [totalCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have at least one node on clearnet
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks LIKE '%ipv%'
+      `;
+      const [clearnetCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have both nodes on Tor 
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks NOT LIKE '%ipv%' AND
+          channels_tmp.networks NOT LIKE '%dns%' AND
+          channels_tmp.networks NOT LIKE '%websocket%'
+      `;
+      const [torCapacity]: any = await DB.query(query);
+
+      const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10);
+      const torCapacityValue = parseInt(torCapacity[0].capacity, 10);
+      const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue;
+
+      return {
+        clearnetCapacity: clearnetCapacityValue,
+        torCapacity: torCapacityValue,
+        unknownCapacity: unknownCapacityValue,
+        ispRanking: ispRanking,
+      };
+    } catch (e) {
+      logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerCountry(countryId: string) {
+    try {
+      const query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
+        WHERE geo_names_country.id = ?
+        ORDER BY capacity DESC
+      `;
+
+      const [rows]: any = await DB.query(query, [countryId]);
+      for (let i = 0; i < rows.length; ++i) {
+        rows[i].country = JSON.parse(rows[i].country);
+        rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows[i].isp = JSON.parse(rows[i].isp);
+      }
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get nodes for country id ${countryId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerISP(ISPId: string) {
+    try {
+      const query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        WHERE nodes.as_number IN (?)
+        ORDER BY capacity DESC
+      `;
+
+      const [rows]: any = await DB.query(query, [ISPId.split(',')]);
+      for (let i = 0; i < rows.length; ++i) {
+        rows[i].country = JSON.parse(rows[i].country);
+        rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+      }
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesCountries() {
+    try {
+      let query = `SELECT geo_names.names as names, geo_names_iso.names as iso_code, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
+        FROM nodes
+        JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        GROUP BY country_id
+        ORDER BY COUNT(DISTINCT nodes.public_key) DESC
+      `;
+      const [nodesCountPerCountry]: any = await DB.query(query);
+
+      query = `SELECT COUNT(*) as total FROM nodes WHERE country_id IS NOT NULL`;
+      const [nodesWithAS]: any = await DB.query(query);
+
+      const nodesPerCountry: any[] = [];
+      for (const country of nodesCountPerCountry) {
+        nodesPerCountry.push({
+          name: JSON.parse(country.names),
+          iso: country.iso_code, 
+          count: country.nodesCount,
+          share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100,
+          capacity: country.capacity,
+        })
+      }
+
+      return nodesPerCountry;
+    } catch (e) {
+      logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  /**
+   * Save or update a node present in the graph
+   */
+  public async $saveNode(node: ILightningApi.Node): Promise<void> {
+    try {
+      const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
+      const query = `INSERT INTO nodes(
+          public_key,
+          first_seen,
+          updated_at,
+          alias,
+          alias_search,
+          color,
+          sockets,
+          status
+        )
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
+        ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
+
+      await DB.query(query, [
+        node.pub_key,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+      ]);
+    } catch (e) {
+      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update node sockets
+   */
+  public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
+    const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
+    try {
+      await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
+    } catch (e) {
+      logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /**
+   * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   */
+   public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
+    if (graphNodesPubkeys.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE nodes
+        SET status = 0
+        WHERE public_key NOT IN (
+          ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
+        )
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private aliasToSearchText(str: string): string {
+    return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '');
+  }
 }
 
 export default new NodesApi();

backend/src/api/explorer/nodes.routes.ts

@@ -1,15 +1,27 @@
 import config from '../../config';
 import { Application, Request, Response } from 'express';
 import nodesApi from './nodes.api';
+import DB from '../../database';
+import { INodesRanking } from '../../mempool.interfaces';
+
 class NodesRoutes {
   constructor() { }
 
   public initRoutes(app: Application) {
     app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/liquidity', this.$getTopNodesByCapacity)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
     ;
   }
 
@@ -22,6 +34,39 @@ class NodesRoutes {
     }
   }
 
+  private async $getNodeGroup(req: Request, res: Response) {
+    try {
+      let nodesList;
+      let nodes: any[] = [];
+      switch (config.MEMPOOL.NETWORK) {
+        case 'testnet':
+          nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
+          break;
+        case 'signet':
+          nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
+          break;
+        default:
+          nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
+      }
+
+      for (let pubKey of nodesList) {
+        try {
+          const node = await nodesApi.$getNode(pubKey);
+          if (node) {
+            nodes.push(node);
+          }
+        } catch (e) {}
+      }
+      
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNode(req: Request, res: Response) {
     try {
       const node = await nodesApi.$getNode(req.params.public_key);
@@ -29,6 +74,9 @@ class NodesRoutes {
         res.status(404).send('Node not found');
         return;
       }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(node);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -38,17 +86,23 @@ class NodesRoutes {
   private async $getHistoricalNodeStats(req: Request, res: Response) {
     try {
       const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(statistics);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
 
-  private async $getTopNodes(req: Request, res: Response) {
+  private async $getNodesRanking(req: Request, res: Response): Promise<void> {
     try {
-      const topCapacityNodes = await nodesApi.$getTopCapacityNodes();
-      const topChannelsNodes = await nodesApi.$getTopChannelsNodes();
-      res.json({
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);
+      const topChannelsNodes = await nodesApi.$getTopChannelsNodes(false);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(<INodesRanking>{
         topByCapacity: topCapacityNodes,
         topByChannels: topChannelsNodes,
       });
@@ -56,6 +110,133 @@ class NodesRoutes {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
+
+  private async $getTopNodesByCapacity(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodesByChannels(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopChannelsNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getOldestNodes(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getOldestNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getISPRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesISPRanking();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getWorldNodes(req: Request, res: Response) {
+    try {
+      const worldNodes = await nodesApi.$getWorldNodes();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(worldNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesPerCountry(req: Request, res: Response) {
+    try {
+      const [country]: any[] = await DB.query(
+        `SELECT geo_names.id, geo_names_country.names as country_names
+        FROM geo_names
+        JOIN geo_names geo_names_country on geo_names.id = geo_names_country.id AND geo_names_country.type = 'country'
+        WHERE geo_names.type = 'country_iso_code' AND geo_names.names = ?`,
+        [req.params.country]
+      );
+
+      if (country.length === 0) {
+        res.status(404).send(`This country does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerCountry(country[0].id);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        country: JSON.parse(country[0].country_names),
+        nodes: nodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesPerISP(req: Request, res: Response) {
+    try {
+      const [isp]: any[] = await DB.query(
+        `SELECT geo_names.names as isp_name
+        FROM geo_names
+        WHERE geo_names.type = 'as_organization' AND geo_names.id = ?`,
+        [req.params.isp]
+      );
+
+      if (isp.length === 0) {
+        res.status(404).send(`This ISP does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerISP(req.params.isp);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        isp: JSON.parse(isp[0].isp_name),
+        nodes: nodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesCountries(req: Request, res: Response) {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesCountries();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 }
 
 export default new NodesRoutes();

backend/src/api/explorer/statistics.api.ts

@@ -6,14 +6,15 @@ class StatisticsApi {
   public async $getStatistics(interval: string | null = null): Promise<any> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, node_count, total_capacity, tor_nodes, clearnet_nodes, unannounced_nodes
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
+      tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
       FROM lightning_stats`;
 
     if (interval) {
       query += ` WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
     }
 
-    query += ` ORDER BY id DESC`;
+    query += ` ORDER BY added DESC`;
 
     try {
       const [rows]: any = await DB.query(query);
@@ -26,8 +27,8 @@ class StatisticsApi {
 
   public async $getLatestStatistics(): Promise<any> {
     try {
-      const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1`);
-      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1 OFFSET 7`);
+      const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
       return {
         latest: rows[0],
         previous: rows2[0],

backend/src/api/fetch-version.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from "path";
+const { spawnSync } = require('child_process');
+
+function getVersion(): string {
+  const packageJson = fs.readFileSync('package.json').toString();
+  return JSON.parse(packageJson).version;
+}
+
+function getGitCommit(): string {
+  if (process.env.MEMPOOL_COMMIT_HASH) {
+    return process.env.MEMPOOL_COMMIT_HASH;
+  } else {
+    const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+    if (!gitRevParse.error) {
+      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+      if (output) {
+        return output;
+      } else {
+        console.log('Could not fetch git commit: No repo available');
+      }
+    } else if (gitRevParse.error.code === 'ENOENT') {
+      console.log('Could not fetch git commit: Command `git` is unavailable');
+    }
+  }
+  return '?';
+}
+
+const versionInfo = {
+  version: getVersion(),
+  gitCommit: getGitCommit()
+}
+
+fs.writeFileSync(
+  path.join(__dirname, 'version.json'),
+  JSON.stringify(versionInfo, null, 2) + "\n"
+);

backend/src/api/lightning/clightning/clightning-client.ts

@@ -0,0 +1,272 @@
+// Imported from https://github.com/shesek/lightning-client-js
+
+'use strict';
+
+const methods = [
+  'addgossip',
+  'autocleaninvoice',
+  'check',
+  'checkmessage',
+  'close',
+  'connect',
+  'createinvoice',
+  'createinvoicerequest',
+  'createoffer',
+  'createonion',
+  'decode',
+  'decodepay',
+  'delexpiredinvoice',
+  'delinvoice',
+  'delpay',
+  'dev-listaddrs',
+  'dev-rescan-outputs',
+  'disableoffer',
+  'disconnect',
+  'estimatefees',
+  'feerates',
+  'fetchinvoice',
+  'fundchannel',
+  'fundchannel_cancel',
+  'fundchannel_complete',
+  'fundchannel_start',
+  'fundpsbt',
+  'getchaininfo',
+  'getinfo',
+  'getlog',
+  'getrawblockbyheight',
+  'getroute',
+  'getsharedsecret',
+  'getutxout',
+  'help',
+  'invoice',
+  'keysend',
+  'legacypay',
+  'listchannels',
+  'listconfigs',
+  'listforwards',
+  'listfunds',
+  'listinvoices',
+  'listnodes',
+  'listoffers',
+  'listpays',
+  'listpeers',
+  'listsendpays',
+  'listtransactions',
+  'multifundchannel',
+  'multiwithdraw',
+  'newaddr',
+  'notifications',
+  'offer',
+  'offerout',
+  'openchannel_abort',
+  'openchannel_bump',
+  'openchannel_init',
+  'openchannel_signed',
+  'openchannel_update',
+  'pay',
+  'payersign',
+  'paystatus',
+  'ping',
+  'plugin',
+  'reserveinputs',
+  'sendinvoice',
+  'sendonion',
+  'sendonionmessage',
+  'sendpay',
+  'sendpsbt',
+  'sendrawtransaction',
+  'setchannelfee',
+  'signmessage',
+  'signpsbt',
+  'stop',
+  'txdiscard',
+  'txprepare',
+  'txsend',
+  'unreserveinputs',
+  'utxopsbt',
+  'waitanyinvoice',
+  'waitblockheight',
+  'waitinvoice',
+  'waitsendpay',
+  'withdraw'
+];
+
+
+import EventEmitter from 'events';
+import { existsSync, statSync } from 'fs';
+import { createConnection, Socket } from 'net';
+import { homedir } from 'os';
+import path from 'path';
+import { createInterface, Interface } from 'readline';
+import logger from '../../../logger';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
+
+class LightningError extends Error {
+  type: string = 'lightning';
+  message: string = 'lightning-client error';
+
+  constructor(error) {
+    super();
+    this.type = error.type;
+    this.message = error.message;
+  }
+}
+
+const defaultRpcPath = path.join(homedir(), '.lightning')
+  , fStat = (...p) => statSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p))
+
+export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
+  private rpcPath: string;
+  private reconnectWait: number;
+  private reconnectTimeout;
+  private reqcount: number;
+  private client: Socket;
+  private rl: Interface;
+  private clientConnectionPromise: Promise<unknown>;
+
+  constructor(rpcPath = defaultRpcPath) {
+    if (!path.isAbsolute(rpcPath)) {
+      throw new Error('The rpcPath must be an absolute path');
+    }
+
+    if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
+      // network directory provided, use the lightning-rpc within in
+      if (fExists(rpcPath, 'lightning-rpc')) {
+        rpcPath = path.join(rpcPath, 'lightning-rpc');
+      }
+
+      // main data directory provided, default to using the bitcoin mainnet subdirectory
+      // to be removed in v0.2.0
+      else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
+        logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`)
+        logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`)
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+      }
+    }
+
+    logger.debug(`[CLightningClient] Connecting to ${rpcPath}`);
+
+    super();
+    this.rpcPath = rpcPath;
+    this.reconnectWait = 0.5;
+    this.reconnectTimeout = null;
+    this.reqcount = 0;
+
+    const _self = this;
+
+    this.client = createConnection(rpcPath).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+    this.rl = createInterface({ input: this.client }).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+
+    this.clientConnectionPromise = new Promise<void>(resolve => {
+      _self.client.on('connect', () => {
+        logger.info(`[CLightningClient] Lightning client connected`);
+        _self.reconnectWait = 1;
+        resolve();
+      });
+
+      _self.client.on('end', () => {
+        logger.err('[CLightningClient] Lightning client connection closed, reconnecting');
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+
+      _self.client.on('error', error => {
+        logger.err(`[CLightningClient] Lightning client connection error: ${error}`);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+    });
+
+    this.rl.on('line', line => {
+      line = line.trim();
+      if (!line) {
+        return;
+      }
+      const data = JSON.parse(line);
+      // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`);
+      _self.emit('res:' + data.id, data);
+    });
+  }
+
+  increaseWaitTime(): void {
+    if (this.reconnectWait >= 16) {
+      this.reconnectWait = 16;
+    } else {
+      this.reconnectWait *= 2;
+    }
+  }
+
+  reconnect(): void {
+    const _self = this;
+
+    if (this.reconnectTimeout) {
+      return;
+    }
+
+    this.reconnectTimeout = setTimeout(() => {
+      logger.debug('[CLightningClient] Trying to reconnect...');
+
+      _self.client.connect(_self.rpcPath);
+      _self.reconnectTimeout = null;
+    }, this.reconnectWait * 1000);
+  }
+
+  call(method, args = []): Promise<any> {
+    const _self = this;
+
+    const callInt = ++this.reqcount;
+    const sendObj = {
+      jsonrpc: '2.0',
+      method,
+      params: args,
+      id: '' + callInt
+    };
+
+    // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`);
+
+    // Wait for the client to connect
+    return this.clientConnectionPromise
+      .then(() => new Promise((resolve, reject) => {
+        // Wait for a response
+        this.once('res:' + callInt, res => res.error == null
+          ? resolve(res.result)
+          : reject(new LightningError(res.error))
+        );
+
+        // Send the command
+        _self.client.write(JSON.stringify(sendObj));
+      }));
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const listnodes: any[] = await this.call('listnodes');
+    const listchannels: any[] = await this.call('listchannels');
+    const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
+
+    return {
+      nodes: listnodes['nodes'].map(node => convertNode(node)),
+      edges: channelsList,
+    };
+  }
+}
+
+const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
+
+methods.forEach(k => {
+  CLightningClient.prototype[protify(k)] = function (...args: any) {
+    return this.call(k, args);
+  };
+});

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -0,0 +1,135 @@
+import { ILightningApi } from '../lightning-api.interface';
+import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
+import logger from '../../../logger';
+import { Common } from '../../common';
+
+/**
+ * Convert a clightning "listnode" entry to a lnd node entry
+ */
+export function convertNode(clNode: any): ILightningApi.Node {
+  return {
+    alias: clNode.alias ?? '',
+    color: `#${clNode.color ?? ''}`,
+    features: [], // TODO parse and return clNode.feature
+    pub_key: clNode.nodeid,
+    addresses: clNode.addresses?.map((addr) => {
+      let address = addr.address;
+      if (addr.type === 'ipv6') {
+        address = `[${address}]`;
+      }
+      return {
+        network: addr.type,
+        addr: `${address}:${addr.port}`
+      };
+    }) ?? [],
+    last_update: clNode?.last_timestamp ?? 0,
+  };
+}
+
+/**
+ * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ */
+export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
+  logger.info('Converting clightning nodes and channels to lnd graph format');
+
+  let loggerTimer = new Date().getTime() / 1000;
+  let channelProcessed = 0;
+
+  const consolidatedChannelList: ILightningApi.Channel[] = [];
+  const clChannelsDict = {};
+  const clChannelsDictCount = {};
+
+  for (const clChannel of clChannels) {
+    if (!clChannelsDict[clChannel.short_channel_id]) {
+      clChannelsDict[clChannel.short_channel_id] = clChannel;
+      clChannelsDictCount[clChannel.short_channel_id] = 1;
+    } else {
+      consolidatedChannelList.push(
+        await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id])
+      );
+      delete clChannelsDict[clChannel.short_channel_id];
+      clChannelsDictCount[clChannel.short_channel_id]++;
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    ++channelProcessed;
+  }
+
+  channelProcessed = 0;
+  const keys = Object.keys(clChannelsDict);
+  for (const short_channel_id of keys) {
+    consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+  }
+
+  return consolidatedChannelList;
+}
+
+/**
+ * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy for both nodes
+ */
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel> {
+  const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
+
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
+  const parts = clChannelA.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
+    capacity: clChannelA.satoshis,
+    last_update: lastUpdate,
+    node1_policy: convertPolicy(clChannelA),
+    node2_policy: convertPolicy(clChannelB),
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannelA.source,
+    node2_pub: clChannelB.source,
+  };
+}
+
+/**
+ * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy of only one node
+ */
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel> {
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
+  const parts = clChannel.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
+    capacity: clChannel.satoshis,
+    last_update: clChannel.last_update ?? 0,
+    node1_policy: convertPolicy(clChannel),
+    node2_policy: null,
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannel.source,
+    node2_pub: clChannel.destination,
+  };
+}
+
+/**
+ * Convert a clightning "listnode" response to a lnd channel policy format
+ */
+function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
+  return {
+    time_lock_delta: clChannel.delay,
+    min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
+    max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
+    fee_base_msat: clChannel.base_fee_millisatoshi,
+    fee_rate_milli_msat: clChannel.fee_per_millionth,
+    disabled: !clChannel.active,
+    last_update: clChannel.last_update ?? 0,
+  };
+}

backend/src/api/lightning/lightning-api-abstract-factory.ts

@@ -1,7 +1,5 @@
 import { ILightningApi } from './lightning-api.interface';
 
 export interface AbstractLightningApi {
-  $getNetworkInfo(): Promise<ILightningApi.NetworkInfo>;
   $getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
-  $getInfo(): Promise<ILightningApi.Info>;
 }

backend/src/api/lightning/lightning-api-factory.ts

@@ -1,9 +1,12 @@
 import config from '../../config';
+import CLightningClient from './clightning/clightning-client';
 import { AbstractLightningApi } from './lightning-api-abstract-factory';
 import LndApi from './lnd/lnd-api';
 
 function lightningApiFactory(): AbstractLightningApi {
-  switch (config.LIGHTNING.BACKEND) {
+  switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
+    case 'cln':
+      return new CLightningClient(config.CLIGHTNING.SOCKET);
     case 'lnd':
     default:
       return new LndApi();

backend/src/api/lightning/lightning-api.interface.ts

@@ -1,71 +1,85 @@
 export namespace ILightningApi {
   export interface NetworkInfo {
-    average_channel_size: number;
-    channel_count: number;
-    max_channel_size: number;
-    median_channel_size: number;
-    min_channel_size: number;
-    node_count: number;
-    not_recently_updated_policy_count: number;
-    total_capacity: number;
+    graph_diameter: number;
+    avg_out_degree: number;
+    max_out_degree: number;
+    num_nodes: number;
+    num_channels: number;
+    total_network_capacity: string;
+    avg_channel_size: number;
+    min_channel_size: string;
+    max_channel_size: string;
+    median_channel_size_sat: string;
+    num_zombie_chans: string;
   }
 
   export interface NetworkGraph {
-    channels: Channel[];
     nodes: Node[];
+    edges: Channel[];
   }
 
   export interface Channel {
-    id: string;
-    capacity: number;
-    policies: Policy[];
-    transaction_id: string;
-    transaction_vout: number;
-    updated_at?: string;
+    channel_id: string;
+    chan_point: string;
+    last_update: number;
+    node1_pub: string;
+    node2_pub: string;
+    capacity: string;
+    node1_policy: RoutingPolicy | null;
+    node2_policy: RoutingPolicy | null;
   }
 
-  interface Policy {
-    public_key: string;
-    base_fee_mtokens?: string;
-    cltv_delta?: number;
-    fee_rate?: number;
-    is_disabled?: boolean;
-    max_htlc_mtokens?: string;
-    min_htlc_mtokens?: string;
-    updated_at?: string;
+  export interface RoutingPolicy {
+    time_lock_delta: number;
+    min_htlc: string;
+    fee_base_msat: string;
+    fee_rate_milli_msat: string;
+    disabled: boolean;
+    max_htlc_msat: string;
+    last_update: number;
   }
 
   export interface Node {
+    last_update: number;
+    pub_key: string;
     alias: string;
+    addresses: {
+      network: string;
+      addr: string;
+    }[];
     color: string;
-    features: Feature[];
-    public_key: string;
-    sockets: string[];
-    updated_at?: string;
+    features: { [key: number]: Feature };
   }
 
   export interface Info {
-    chains: string[];
-    color: string;
-    active_channels_count: number;
+    identity_pubkey: string;
     alias: string;
-    current_block_hash: string;
-    current_block_height: number;
-    features: Feature[];
-    is_synced_to_chain: boolean;
-    is_synced_to_graph: boolean;
-    latest_block_at: string;
-    peers_count: number;
-    pending_channels_count: number;
-    public_key: string;
-    uris: any[];
+    num_pending_channels: number;
+    num_active_channels: number;
+    num_peers: number;
+    block_height: number;
+    block_hash: string;
+    synced_to_chain: boolean;
+    testnet: boolean;
+    uris: string[];
+    best_header_timestamp: string;
     version: string;
+    num_inactive_channels: number;
+    chains: {
+      chain: string;
+      network: string;
+    }[];
+    color: string;
+    synced_to_graph: boolean;
+    features: { [key: number]: Feature };
+    commit_hash: string;
+    /** Available on LND since v0.15.0-beta */
+    require_htlc_interceptor?: boolean;
   }
-  
+
   export interface Feature {
-    bit: number;
-    is_known: boolean;
+    name: string;
     is_required: boolean;
-    type?: string;
+    is_known: boolean;
   }
-}
+}

backend/src/api/lightning/lnd/lnd-api.ts

@@ -1,44 +1,40 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { Agent } from 'https';
+import * as fs from 'fs';
 import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
-import * as fs from 'fs';
-import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning';
 import config from '../../../config';
-import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
-  private lnd: any;
+  axiosConfig: AxiosRequestConfig = {};
+
   constructor() {
-    if (!config.LIGHTNING.ENABLED) {
-      return;
-    }
-    try {
-      const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64');
-      const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64');
-
-      const { lnd } = authenticatedLndGrpc({
-        cert: tls,
-        macaroon: macaroon,
-        socket: config.LND.SOCKET,
-      });
-
-      this.lnd = lnd;
-    } catch (e) {
-      logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
-      process.exit(1);
+    if (config.LIGHTNING.ENABLED) {
+      this.axiosConfig = {
+        headers: {
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
+        },
+        httpsAgent: new Agent({
+          ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
+        }),
+        timeout: 10000
+      };
     }
   }
 
   async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
-    return await getNetworkInfo({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getInfo(): Promise<ILightningApi.Info> {
-    // @ts-ignore
-    return await getWalletInfo({ lnd: this.lnd });
+    return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
-    return await getNetworkGraph({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+      .then((response) => response.data);
   }
 }
 

backend/src/api/mining/mining-routes.ts

@@ -26,7 +26,8 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
-      ;
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
+    ;
   }
 
   private async $getPool(req: Request, res: Response): Promise<void> {
@@ -233,6 +234,18 @@ class MiningRoutes {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
+
+  public async $getBlockAudit(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 }
 
 export default new MiningRoutes();

backend/src/api/mining/mining.ts

@@ -473,7 +473,7 @@ class Mining {
 
       for (const block of blocksWithoutPrices) {
         // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
-        if (block.height < 68951) {
+        if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
           blocksPrices.push({
             height: block.height,
             priceId: prices[0].id,
@@ -492,11 +492,11 @@ class Mining {
 
         if (blocksPrices.length >= 100000) {
           totalInserted += blocksPrices.length;
+          let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
           if (blocksWithoutPrices.length > 200000) {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-          } else {
-            logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+            logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
           }
+          logger.debug(logStr);
           await BlocksRepository.$saveBlockPrices(blocksPrices);
           blocksPrices.length = 0;
         }
@@ -504,11 +504,11 @@ class Mining {
 
       if (blocksPrices.length > 0) {
         totalInserted += blocksPrices.length;
+        let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
         if (blocksWithoutPrices.length > 200000) {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
-        } else {
-          logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
+          logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
         }
+        logger.debug(logStr);
         await BlocksRepository.$saveBlockPrices(blocksPrices);
       }
     } catch (e) {

backend/src/api/websocket-handler.ts

@@ -451,9 +451,12 @@ class WebsocketHandler {
             value: tx.value,
           };
         });  
-        BlocksSummariesRepository.$saveSummary(block.height, null, {
-          id: block.id,
-          transactions: stripped
+        BlocksSummariesRepository.$saveSummary({
+          height: block.height,
+          template: {
+            id: block.id,
+            transactions: stripped
+          }
         });
 
         BlocksAuditsRepository.$saveAudit({

backend/src/config.ts

@@ -1,4 +1,6 @@
-const configFile = require('../mempool-config.json');
+const configFromFile = require(
+    process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
+);
 
 interface IConfig {
   MEMPOOL: {
@@ -24,6 +26,8 @@ interface IConfig {
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
     AUTOMATIC_BLOCK_REINDEXING: boolean;
+    POOLS_JSON_URL: string,
+    POOLS_JSON_TREE_URL: string,
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -31,10 +35,17 @@ interface IConfig {
   LIGHTNING: {
     ENABLED: boolean;
     BACKEND: 'lnd' | 'cln' | 'ldk';
+    TOPOLOGY_FOLDER: string;
+    STATS_REFRESH_INTERVAL: number;
+    GRAPH_REFRESH_INTERVAL: number;
+    LOGGER_UPDATE_INTERVAL: number;
   };
   LND: {
     TLS_CERT_PATH: string;
     MACAROON_PATH: string;
+    REST_API_URL: string;
+  };
+  CLIGHTNING: {
     SOCKET: string;
   };
   ELECTRUM: {
@@ -102,6 +113,7 @@ interface IConfig {
     ENABLED: boolean;
     GEOLITE2_CITY: string;
     GEOLITE2_ASN: string;
+    GEOIP2_ISP: string;
   },
 }
 
@@ -129,6 +141,8 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -176,12 +190,19 @@ const defaults: IConfig = {
   },
   'LIGHTNING': {
     'ENABLED': false,
-    'BACKEND': 'lnd'
+    'BACKEND': 'lnd',
+    'TOPOLOGY_FOLDER': '',
+    'STATS_REFRESH_INTERVAL': 600,
+    'GRAPH_REFRESH_INTERVAL': 600,
+    'LOGGER_UPDATE_INTERVAL': 30,
   },
   'LND': {
     'TLS_CERT_PATH': '',
     'MACAROON_PATH': '',
-    'SOCKET': 'localhost:10009',
+    'REST_API_URL': 'https://localhost:8080',
+  },
+  'CLIGHTNING': {
+    'SOCKET': '',
   },
   'SOCKS5PROXY': {
     'ENABLED': false,
@@ -206,7 +227,8 @@ const defaults: IConfig = {
   "MAXMIND": {
     'ENABLED': false,
     "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
   },
 };
 
@@ -222,13 +244,14 @@ class Config implements IConfig {
   BISQ: IConfig['BISQ'];
   LIGHTNING: IConfig['LIGHTNING'];
   LND: IConfig['LND'];
+  CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
 
   constructor() {
-    const configs = this.merge(configFile, defaults);
+    const configs = this.merge(configFromFile, defaults);
     this.MEMPOOL = configs.MEMPOOL;
     this.ESPLORA = configs.ESPLORA;
     this.ELECTRUM = configs.ELECTRUM;
@@ -240,6 +263,7 @@ class Config implements IConfig {
     this.BISQ = configs.BISQ;
     this.LIGHTNING = configs.LIGHTNING;
     this.LND = configs.LND;
+    this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;

backend/src/database.ts

@@ -1,7 +1,7 @@
 import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
-import { PoolOptions } from 'mysql2/typings/mysql';
+import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
 
  class DB {
   constructor() {
@@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql';
     }
   }
 
-  public async query(query, params?) {
+  public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
+    OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
+  {
     this.checkDBFlag();
     const pool = await this.getPool();
     return pool.query(query, params);

backend/src/index.ts

@@ -28,12 +28,13 @@ import nodesRoutes from './api/explorer/nodes.routes';
 import channelsRoutes from './api/explorer/channels.routes';
 import generalLightningRoutes from './api/explorer/general.routes';
 import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
-import nodeSyncService from './tasks/lightning/node-sync.service';
-import statisticsRoutes from "./api/statistics/statistics.routes";
-import miningRoutes from "./api/mining/mining-routes";
-import bisqRoutes from "./api/bisq/bisq.routes";
-import liquidRoutes from "./api/liquid/liquid.routes";
-import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
+import networkSyncService from './tasks/lightning/network-sync.service';
+import statisticsRoutes from './api/statistics/statistics.routes';
+import miningRoutes from './api/mining/mining-routes';
+import bisqRoutes from './api/bisq/bisq.routes';
+import liquidRoutes from './api/liquid/liquid.routes';
+import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
+import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher";
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -136,8 +137,7 @@ class Server {
     }
 
     if (config.LIGHTNING.ENABLED) {
-      nodeSyncService.$startService()
-        .then(() => lightningStatsUpdater.$startService());
+      this.$runLightningBackend();
     }
 
     this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
@@ -183,6 +183,18 @@ class Server {
     }
   }
 
+  async $runLightningBackend() {
+    try {
+      await fundingTxFetcher.$init();
+      await networkSyncService.$startService();
+      await lightningStatsUpdater.$startService();
+    } catch(e) {
+      logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      await Common.sleep$(1000 * 60);
+      this.$runLightningBackend();
+    };
+}
+
   setUpWebsocketHandling() {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);

backend/src/indexer.ts

@@ -6,13 +6,12 @@ import logger from './logger';
 import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
+import PricesRepository from './repositories/PricesRepository';
 
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
-
-  constructor() {
-  }
+  tasksRunning: string[] = [];
 
   public reindex() {
     if (Common.indexingEnabled()) {
@@ -20,6 +19,28 @@ class Indexer {
     }
   }
 
+  public async runSingleTask(task: 'blocksPrices') {
+    if (!Common.indexingEnabled()) {
+      return;
+    }
+
+    if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      const lastestPriceId = await PricesRepository.$getLatestPriceId();
+      if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        setTimeout(() => {
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.runSingleTask('blocksPrices');
+        }, 10000);
+      } else {
+        logger.debug(`Blocks prices indexer will run now`)
+        await mining.$indexBlockPrices();
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+      }
+    }
+  }
+
   public async $run() {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
@@ -50,7 +71,7 @@ class Indexer {
         return;
       }
 
-      await mining.$indexBlockPrices();
+      this.runSingleTask('blocksPrices');
       await mining.$indexDifficultyAdjustments();
       await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
       await mining.$generateNetworkHashrateHistory();

backend/src/logger.ts

@@ -74,7 +74,7 @@ class Logger {
 
   private getNetwork(): string {
     if (config.LIGHTNING.ENABLED) {
-      return 'lightning';
+      return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; 
     }
     if (config.BISQ.ENABLED) {
       return 'bisq';

backend/src/mempool.interfaces.ts

@@ -251,3 +251,41 @@ export interface RewardStats {
   totalFee: number;
   totalTx: number;
 }
+
+export interface ITopNodesPerChannels {
+  publicKey: string,
+  alias: string,
+  channels?: number,
+  capacity: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface ITopNodesPerCapacity {
+  publicKey: string,
+  alias: string,
+  capacity: number,
+  channels?: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface INodesRanking {
+  topByCapacity: ITopNodesPerCapacity[];
+  topByChannels: ITopNodesPerChannels[];
+}
+
+export interface IOldestNodes {
+  publicKey: string,
+  alias: string,
+  firstSeen: number,
+  channels?: number,
+  capacity: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,4 +1,3 @@
-import transactionUtils from '../api/transaction-utils';
 import DB from '../database';
 import logger from '../logger';
 import { BlockAudit } from '../mempool.interfaces';

backend/src/repositories/BlocksSummariesRepository.ts

@@ -17,18 +17,18 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(height: number, mined: BlockSummary | null = null, template: BlockSummary | null = null) {
-    const blockId = mined?.id ?? template?.id;
+  public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) {
+    const blockId = params.mined?.id ?? params.template?.id;
     try {
       const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`);
       if (dbSummary.length === 0) { // First insertion
         await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [
-          height, blockId, JSON.stringify(mined?.transactions ?? []), JSON.stringify(template?.transactions ?? [])
+          params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? [])
         ]);
-      } else if (mined !== null) { // Update mined block summary
-        await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${mined.id}"`, [JSON.stringify(mined?.transactions)]);
-      } else if (template !== null) { // Update template block summary
-        await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${template.id}"`, [JSON.stringify(template?.transactions)]);
+      } else if (params.mined !== undefined) { // Update mined block summary
+        await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]);
+      } else if (params.template !== undefined) { // Update template block summary
+        await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]);
       }
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart

backend/src/repositories/NodesSocketsRepository.ts

@@ -0,0 +1,45 @@
+import { ResultSetHeader } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeSocket {
+  publicKey: string;
+  network: string | null;
+  addr: string;
+}
+
+class NodesSocketsRepository {
+  public async $saveSocket(socket: NodeSocket): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO nodes_sockets(public_key, socket, type)
+        VALUE (?, ?, ?)
+      `, [socket.publicKey, socket.addr, socket.network]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes sockets
+      }
+    }
+   }
+
+   public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
+    if (addresses.length === 0) {
+      return 0;
+    }
+    try {
+      const query = `
+        DELETE FROM nodes_sockets
+        WHERE public_key = ?
+        AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
+      `;
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+   }
+}
+
+export default new NodesSocketsRepository();

backend/src/repositories/PricesRepository.ts

@@ -27,6 +27,11 @@ class PricesRepository {
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
+  public async $getLatestPriceId(): Promise<number | null> {
+    const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].id : null;
+  }
+
   public async $getLatestPriceTime(): Promise<number> {
     const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;

backend/src/tasks/lightning/network-sync.service.ts

@@ -0,0 +1,431 @@
+import DB from '../../database';
+import logger from '../../logger';
+import channelsApi from '../../api/explorer/channels.api';
+import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
+import config from '../../config';
+import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
+import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+import { $lookupNodeLocation } from './sync-tasks/node-locations';
+import lightningApi from '../../api/lightning/lightning-api-factory';
+import nodesApi from '../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
+import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
+import { Common } from '../../api/common';
+import blocks from '../../api/blocks';
+
+class NetworkSyncService {
+  loggerTimer = 0;
+  closedChannelsScanBlock = 0;
+
+  constructor() {}
+
+  public async $startService(): Promise<void> {
+    logger.info('Starting lightning network sync service');
+
+    this.loggerTimer = new Date().getTime() / 1000;
+
+    await this.$runTasks();
+  }
+
+  private async $runTasks(): Promise<void> {
+    try {
+      logger.info(`Updating nodes and channels`);
+
+      const networkGraph = await lightningApi.$getNetworkGraph();
+      if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
+        logger.info(`LN Network graph is empty, retrying in 10 seconds`);
+        setTimeout(() => { this.$runTasks(); }, 10000);
+        return;
+      }
+
+      await this.$updateNodesList(networkGraph.nodes);
+      await this.$updateChannelsList(networkGraph.edges);
+      await this.$deactivateChannelsWithoutActiveNodes();
+      await this.$lookUpCreationDateFromChain();
+      await this.$updateNodeFirstSeen();
+      await this.$scanForClosedChannels();
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        await this.$runClosedChannelsForensics();
+      }
+
+    } catch (e) {
+      logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
+    }
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
+  }
+
+  /**
+   * Update the `nodes` table to reflect the current network graph state
+   */
+  private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
+    let progress = 0;
+
+    let deletedSockets = 0;
+    const graphNodesPubkeys: string[] = [];
+    for (const node of nodes) {
+      const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
+      node.last_update = Math.max(node.last_update, latestUpdated);
+
+      await nodesApi.$saveNode(node);
+      graphNodesPubkeys.push(node.pub_key);
+      ++progress;
+
+      const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+      if (elapsedSeconds > 10) {
+        logger.info(`Updating node ${progress}/${nodes.length}`);
+        this.loggerTimer = new Date().getTime() / 1000;
+      }
+
+      const addresses: string[] = [];
+      for (const socket of node.addresses) {
+        await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket));
+        addresses.push(socket.addr);
+      }
+      deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+    }
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
+
+    // If a channel if not present in the graph, mark it as inactive
+    await nodesApi.$setNodesInactive(graphNodesPubkeys);
+
+    if (config.MAXMIND.ENABLED) {
+      $lookupNodeLocation();
+    }
+  }
+
+  /**
+   * Update the `channels` table to reflect the current network graph state
+   */
+  private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
+    try {
+      const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`);
+      const closedChannels = {};
+      for (const closedChannel of closedChannelsRaw) {
+        closedChannels[closedChannel.id] = true;
+      }
+
+      let progress = 0;
+
+      const graphChannelsIds: string[] = [];
+      for (const channel of channels) {
+        if (!closedChannels[channel.channel_id]) {
+          await channelsApi.$saveChannel(channel);
+        }
+        graphChannelsIds.push(channel.channel_id);
+        ++progress;
+
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      logger.info(`${progress} channels updated`);
+
+      // If a channel if not present in the graph, mark it as inactive
+      await channelsApi.$setChannelsInactive(graphChannelsIds);
+    } catch (e) {
+      logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
+    }
+  }
+
+  // This method look up the creation date of the earliest channel of the node
+  // and update the node to that date in order to get the earliest first seen date
+  private async $updateNodeFirstSeen(): Promise<void> {
+    let progress = 0;
+    let updated = 0;
+
+    try {
+      const [nodes]: any[] = await DB.query(`
+        SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node1_public_key = nodes.public_key
+        ) AS created1,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node2_public_key = nodes.public_key
+        ) AS created2
+        FROM nodes
+      `);
+
+      for (const node of nodes) {
+        const lowest = Math.min(
+          node.created1 ?? Number.MAX_SAFE_INTEGER,
+          node.created2 ?? Number.MAX_SAFE_INTEGER,
+          node.first_seen ?? Number.MAX_SAFE_INTEGER
+        );
+        if (lowest < node.first_seen) {
+          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
+          const params = [lowest, node.public_key];
+          await DB.query(query, params);
+        }
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+          ++updated;
+        }
+      }
+      logger.info(`Updated ${updated} node first seen dates`);
+    } catch (e) {
+      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $lookUpCreationDateFromChain(): Promise<void> {
+    let progress = 0;
+
+    logger.info(`Running channel creation date lookup`);
+    try {
+      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
+      for (const channel of channels) {
+        const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
+        await DB.query(`
+          UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
+          [transaction.timestamp, channel.id]
+        );
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel creation date ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Updated ${channels.length} channels' creation date`);
+    } catch (e) {
+      logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * If a channel does not have any active node linked to it, then also
+   * mark that channel as inactive
+   */
+  private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
+    logger.info(`Find channels which nodes are offline`);
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE channels.status = 1
+        AND (
+          (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node1_public_key
+            AND nodes.status = 1
+          ) = 0
+        OR (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node2_public_key
+            AND nodes.status = 1
+          ) = 0)
+        `);
+
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      }
+    } catch (e) {
+      logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $scanForClosedChannels(): Promise<void> {
+    if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
+      logger.debug(`We've already scan closed channels for this block, skipping.`);
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      let log = `Starting closed channels scan`;
+      if (this.closedChannelsScanBlock > 0) {
+        log += `. Last scan was at block ${this.closedChannelsScanBlock}`;
+      } else {
+        log += ` for the first time`;
+      }
+      logger.info(log);
+
+      const channels = await channelsApi.$getChannelsByStatus([0, 1]);
+      for (const channel of channels) {
+        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
+        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
+          logger.debug('Marking channel: ' + channel.id + ' as closed.');
+          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
+            [spendingTx.status.block_time, channel.id]);
+          if (spendingTx.txid && !channel.closing_transaction_id) {
+            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+          }
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
+      logger.info(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`);
+    } catch (e) {
+      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /*
+    1. Mutually closed
+    2. Forced closed
+    3. Forced closed with penalty
+  */
+
+  private async $runClosedChannelsForensics(): Promise<void> {
+    if (!config.ESPLORA.REST_API_URL) {
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      logger.info(`Started running closed channel forensics...`);
+      const channels = await channelsApi.$getClosedChannelsWithoutReason();
+      for (const channel of channels) {
+        let reason = 0;
+        // Only Esplora backend can retrieve spent transaction outputs
+        try {
+          let outspends: IEsploraApi.Outspend[] | undefined;
+          try {
+            outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
+          } catch (e) {
+            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+            continue;
+          }
+          const lightningScriptReasons: number[] = [];
+          for (const outspend of outspends) {
+            if (outspend.spent && outspend.txid) {
+              let spendingTx: IEsploraApi.Transaction | undefined;
+              try {
+                spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
+              } catch (e) {
+                logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
+                continue;
+              }
+              const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              lightningScriptReasons.push(lightningScript);
+            }
+          }
+          if (lightningScriptReasons.length === outspends.length
+            && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
+            reason = 1;
+          } else {
+            const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
+            if (filteredReasons.length) {
+              if (filteredReasons.some((r) => r === 2 || r === 4)) {
+                reason = 3;
+              } else {
+                reason = 2;
+              }
+            } else {
+              /*
+                We can detect a commitment transaction (force close) by reading Sequence and Locktime
+                https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
+              */
+              let closingTx: IEsploraApi.Transaction | undefined;
+              try {
+                closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
+              } catch (e) {
+                logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
+                continue;
+              }
+              const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
+              const locktimeHex: string = closingTx.locktime.toString(16);
+              if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
+                reason = 2; // Here we can't be sure if it's a penalty or not
+              } else {
+                reason = 1;
+              }
+            }
+          }
+          if (reason) {
+            logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
+            await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+          }
+        } catch (e) {
+          logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Closed channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private findLightningScript(vin: IEsploraApi.Vin): number {
+    const topElement = vin.witness[vin.witness.length - 2];
+      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
+        if (topElement === '01') {
+          // top element is '01' to get in the revocation path
+          // 'Revoked Lightning Force Close';
+          // Penalty force closed
+          return 2;
+        } else {
+          // top element is '', this is a delayed to_local output
+          // 'Lightning Force Close';
+          return 3;
+        }
+      } else if (
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
+      ) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
+        if (topElement.length === 66) {
+          // top element is a public key
+          // 'Revoked Lightning HTLC'; Penalty force closed
+          return 4;
+        } else if (topElement) {
+          // top element is a preimage
+          // 'Lightning HTLC';
+          return 5;
+        } else {
+          // top element is '' to get in the expiry of the script
+          // 'Expired Lightning HTLC';
+          return 6;
+        }
+      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
+        if (topElement) {
+          // top element is a signature
+          // 'Lightning Anchor';
+          return 7;
+        } else {
+          // top element is '', it has been swept after 16 blocks
+          // 'Swept Lightning Anchor';
+          return 8;
+        }
+      }
+      return 1;
+  }
+}
+
+export default new NetworkSyncService();

backend/src/tasks/lightning/node-sync.service.ts

@@ -1,403 +0,0 @@
-import { chanNumber } from 'bolt07';
-import DB from '../../database';
-import logger from '../../logger';
-import channelsApi from '../../api/explorer/channels.api';
-import bitcoinClient from '../../api/bitcoin/bitcoin-client';
-import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
-import config from '../../config';
-import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
-import lightningApi from '../../api/lightning/lightning-api-factory';
-import { ILightningApi } from '../../api/lightning/lightning-api.interface';
-import { $lookupNodeLocation } from './sync-tasks/node-locations';
-
-class NodeSyncService {
-  constructor() {}
-
-  public async $startService() {
-    logger.info('Starting node sync service');
-
-    await this.$runUpdater();
-
-    setInterval(async () => {
-      await this.$runUpdater();
-    }, 1000 * 60 * 60);
-  }
-
-  private async $runUpdater() {
-    try {
-      logger.info(`Updating nodes and channels...`);
-
-      const networkGraph = await lightningApi.$getNetworkGraph();
-
-      for (const node of networkGraph.nodes) {
-        await this.$saveNode(node);
-      }
-      logger.info(`Nodes updated.`);
-
-      if (config.MAXMIND.ENABLED) {
-        await $lookupNodeLocation();
-      }
-
-      await this.$setChannelsInactive();
-
-      for (const channel of networkGraph.channels) {
-        await this.$saveChannel(channel);
-      }
-      logger.info(`Channels updated.`);
-
-      await this.$findInactiveNodesAndChannels();
-      await this.$lookUpCreationDateFromChain();
-      await this.$updateNodeFirstSeen();
-      await this.$scanForClosedChannels();
-      if (config.MEMPOOL.BACKEND === 'esplora') {
-        await this.$runClosedChannelsForensics();
-      }
-
-    } catch (e) {
-      logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // This method look up the creation date of the earliest channel of the node
-  // and update the node to that date in order to get the earliest first seen date
-  private async $updateNodeFirstSeen() {
-    try {
-      const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
-      for (const node of nodes) {
-        let lowest = 0;
-        if (node.created1) {
-          if (node.created2 && node.created2 < node.created1) {
-            lowest = node.created2;
-          } else {
-            lowest = node.created1;
-          }
-        } else if (node.created2) {
-          lowest = node.created2;
-        }
-        if (lowest && lowest < node.first_seen) {
-          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
-          const params = [lowest, node.public_key];
-          await DB.query(query, params);
-        }
-      }
-      logger.info(`Node first seen dates scan complete.`);
-    } catch (e) {
-      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $lookUpCreationDateFromChain() {
-    logger.info(`Running channel creation date lookup...`);
-    try {
-      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
-      for (const channel of channels) {
-        const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
-        await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
-      }
-      logger.info(`Channel creation dates scan complete.`);
-    } catch (e) {
-      logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // Looking for channels whos nodes are inactive
-  private async $findInactiveNodesAndChannels(): Promise<void> {
-    logger.info(`Running inactive channels scan...`);
-
-    try {
-      // @ts-ignore
-      const [channels]: [ILightningApi.Channel[]] = await DB.query(`SELECT channels.id FROM channels WHERE channels.status = 1 AND ((SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node1_public_key) = 0 OR (SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node2_public_key) = 0)`);
-
-      for (const channel of channels) {
-        await this.$updateChannelStatus(channel.id, 0);
-      }
-      logger.info(`Inactive channels scan complete.`);
-    } catch (e) {
-      logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $scanForClosedChannels(): Promise<void> {
-    try {
-      logger.info(`Starting closed channels scan...`);
-      const channels = await channelsApi.$getChannelsByStatus(0);
-      for (const channel of channels) {
-        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
-        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
-          logger.debug('Marking channel: ' + channel.id + ' as closed.');
-          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
-            [spendingTx.status.block_time, channel.id]);
-          if (spendingTx.txid && !channel.closing_transaction_id) {
-            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
-          }
-        }
-      }
-      logger.info(`Closed channels scan complete.`);
-    } catch (e) {
-      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  /*
-    1. Mutually closed
-    2. Forced closed
-    3. Forced closed with penalty
-  */
-
-  private async $runClosedChannelsForensics(): Promise<void> {
-    if (!config.ESPLORA.REST_API_URL) {
-      return;
-    }
-    try {
-      logger.info(`Started running closed channel forensics...`);
-      const channels = await channelsApi.$getClosedChannelsWithoutReason();
-      for (const channel of channels) {
-        let reason = 0;
-        // Only Esplora backend can retrieve spent transaction outputs
-        const outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
-        const lightningScriptReasons: number[] = [];
-        for (const outspend of outspends) {
-          if (outspend.spent && outspend.txid) {
-            const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
-            const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
-            lightningScriptReasons.push(lightningScript);
-          }
-        }
-        if (lightningScriptReasons.length === outspends.length
-          && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
-          reason = 1;
-        } else {
-          const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
-          if (filteredReasons.length) {
-            if (filteredReasons.some((r) => r === 2 || r === 4)) {
-              reason = 3;
-            } else {
-              reason = 2;
-            }
-          } else {
-            /*
-              We can detect a commitment transaction (force close) by reading Sequence and Locktime
-              https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
-            */
-            const closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
-            const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
-            const locktimeHex: string = closingTx.locktime.toString(16);
-            if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
-              reason = 2; // Here we can't be sure if it's a penalty or not
-            } else {
-              reason = 1;
-            }
-          }
-        }
-        if (reason) {
-          logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
-          await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
-        }
-      }
-      logger.info(`Closed channels forensics scan complete.`);
-    } catch (e) {
-      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private findLightningScript(vin: IEsploraApi.Vin): number {
-    const topElement = vin.witness[vin.witness.length - 2];
-      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
-        if (topElement === '01') {
-          // top element is '01' to get in the revocation path
-          // 'Revoked Lightning Force Close';
-          // Penalty force closed
-          return 2;
-        } else {
-          // top element is '', this is a delayed to_local output
-          // 'Lightning Force Close';
-          return 3;
-        }
-      } else if (
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
-      ) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
-        if (topElement.length === 66) {
-          // top element is a public key
-          // 'Revoked Lightning HTLC'; Penalty force closed
-          return 4;
-        } else if (topElement) {
-          // top element is a preimage
-          // 'Lightning HTLC';
-          return 5;
-        } else {
-          // top element is '' to get in the expiry of the script
-          // 'Expired Lightning HTLC';
-          return 6;
-        }
-      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
-        if (topElement) {
-          // top element is a signature
-          // 'Lightning Anchor';
-          return 7;
-        } else {
-          // top element is '', it has been swept after 16 blocks
-          // 'Swept Lightning Anchor';
-          return 8;
-        }
-      }
-      return 1;
-  }
-
-  private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
-    const fromChannel = chanNumber({ channel: channel.id }).number;
-
-    try {
-      const query = `INSERT INTO channels
-        (
-          id,
-          short_id,
-          capacity,
-          transaction_id,
-          transaction_vout,
-          updated_at,
-          status,
-          node1_public_key,
-          node1_base_fee_mtokens,
-          node1_cltv_delta,
-          node1_fee_rate,
-          node1_is_disabled,
-          node1_max_htlc_mtokens,
-          node1_min_htlc_mtokens,
-          node1_updated_at,
-          node2_public_key,
-          node2_base_fee_mtokens,
-          node2_cltv_delta,
-          node2_fee_rate,
-          node2_is_disabled,
-          node2_max_htlc_mtokens,
-          node2_min_htlc_mtokens,
-          node2_updated_at
-        )
-        VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        ON DUPLICATE KEY UPDATE
-          capacity = ?,
-          updated_at = ?,
-          status = 1,
-          node1_public_key = ?,
-          node1_base_fee_mtokens = ?,
-          node1_cltv_delta = ?,
-          node1_fee_rate = ?,
-          node1_is_disabled = ?,
-          node1_max_htlc_mtokens = ?,
-          node1_min_htlc_mtokens = ?,
-          node1_updated_at = ?,
-          node2_public_key = ?,
-          node2_base_fee_mtokens = ?,
-          node2_cltv_delta = ?,
-          node2_fee_rate = ?,
-          node2_is_disabled = ?,
-          node2_max_htlc_mtokens = ?,
-          node2_min_htlc_mtokens = ?,
-          node2_updated_at = ?
-        ;`;
-
-      await DB.query(query, [
-        fromChannel,
-        channel.id,
-        channel.capacity,
-        channel.transaction_id,
-        channel.transaction_vout,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-        channel.capacity,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-      ]);
-    } catch (e) {
-      logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> {
-    try {
-      await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]);
-    } catch (e) {
-      logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $setChannelsInactive(): Promise<void> {
-    try {
-      await DB.query(`UPDATE channels SET status = 0 WHERE status = 1`);
-    } catch (e) {
-      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $saveNode(node: ILightningApi.Node): Promise<void> {
-    try {
-      const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00';
-      const sockets = node.sockets.join(',');
-      const query = `INSERT INTO nodes(
-          public_key,
-          first_seen,
-          updated_at,
-          alias,
-          color,
-          sockets
-        )
-        VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
-
-      await DB.query(query, [
-        node.public_key,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-      ]);
-    } catch (e) {
-      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private utcDateToMysql(dateString: string): string {
-    const d = new Date(Date.parse(dateString));
-    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
-  }
-}
-
-export default new NodeSyncService();

backend/src/tasks/lightning/stats-updater.service.ts

@@ -1,318 +1,33 @@
-
-import DB from '../../database';
 import logger from '../../logger';
 import lightningApi from '../../api/lightning/lightning-api-factory';
-import channelsApi from '../../api/explorer/channels.api';
-import * as net from 'net';
+import LightningStatsImporter from './sync-tasks/stats-importer';
+import config from '../../config';
+import { Common } from '../../api/common';
 
 class LightningStatsUpdater {
-  hardCodedStartTime = '2018-01-12';
-
-  public async $startService() {
+  public async $startService(): Promise<void> {
     logger.info('Starting Lightning Stats service');
-    let isInSync = false;
-    let error: any;
-    try {
-      error = null;
-      isInSync = await this.$lightningIsSynced();
-    } catch (e) {
-      error = e;
-    }
-    if (!isInSync) {
-      if (error) {
-        logger.warn('Was not able to fetch Lightning Node status: ' + (error instanceof Error ? error.message : error) + '. Retrying in 1 minute...');
-      } else {
-        logger.notice('The Lightning graph is not yet in sync. Retrying in 1 minute...');
-      }
-      setTimeout(() => this.$startService(), 60 * 1000);
-      return;
-    }
 
-    await this.$populateHistoricalStatistics();
-    await this.$populateHistoricalNodeStatistics();
-
-    setTimeout(() => {
-      this.$runTasks();
-    }, this.timeUntilMidnight());
-  }
-
-  private timeUntilMidnight(): number {
-    const date = new Date();
-    this.setDateMidnight(date);
-    date.setUTCHours(24);
-    return date.getTime() - new Date().getTime();
-  }
-
-  private setDateMidnight(date: Date): void {
-    date.setUTCHours(0);
-    date.setUTCMinutes(0);
-    date.setUTCSeconds(0);
-    date.setUTCMilliseconds(0);
-  }
-
-  private async $lightningIsSynced(): Promise<boolean> {
-    const nodeInfo = await lightningApi.$getInfo();
-    return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph;
+    await this.$runTasks();
+    LightningStatsImporter.$run();
   }
 
   private async $runTasks(): Promise<void> {
-    await this.$logLightningStatsDaily();
-    await this.$logNodeStatsDaily();
+    await this.$logStatsDaily();
 
-    setTimeout(() => {
-      this.$runTasks();
-    }, this.timeUntilMidnight());
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
   }
 
-  private async $logLightningStatsDaily() {
-    try {
-      logger.info(`Running lightning daily stats log...`);  
-
-      const networkGraph = await lightningApi.$getNetworkGraph();
-      let total_capacity = 0;
-      for (const channel of networkGraph.channels) {
-        if (channel.capacity) {
-          total_capacity += channel.capacity;
-        }
-      }
-
-      let clearnetNodes = 0;
-      let torNodes = 0;
-      let unannouncedNodes = 0;
-      for (const node of networkGraph.nodes) {
-        let isUnnanounced = true;
-        for (const socket of node.sockets) {
-          const hasOnion = socket.indexOf('.onion') !== -1;
-          if (hasOnion) {
-            torNodes++;
-            isUnnanounced = false;
-          }
-          const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
-          if (hasClearnet) {
-            clearnetNodes++;
-            isUnnanounced = false;
-          }
-        }
-        if (isUnnanounced) {
-          unannouncedNodes++;
-        }
-      }
-
-      const channelStats = await channelsApi.$getChannelsStats();
-
-      const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes,
-          avg_capacity,
-          avg_fee_rate,
-          avg_base_fee_mtokens,
-          med_capacity,
-          med_fee_rate,
-          med_base_fee_mtokens
-        )
-        VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-      await DB.query(query, [
-        networkGraph.channels.length,
-        networkGraph.nodes.length,
-        total_capacity,
-        torNodes,
-        clearnetNodes,
-        unannouncedNodes,
-        channelStats.avgCapacity,
-        channelStats.avgFeeRate,
-        channelStats.avgBaseFee,
-        channelStats.medianCapacity,
-        channelStats.medianFeeRate,
-        channelStats.medianBaseFee,
-      ]);
-      logger.info(`Lightning daily stats done.`);
-    } catch (e) {
-      logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $logNodeStatsDaily() {
-    try {
-      logger.info(`Running daily node stats update...`);
-
-      const query = `SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, c2.channels_capacity_right FROM nodes LEFT JOIN (SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left FROM channels WHERE channels.status < 2 GROUP BY node1_public_key) c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN (SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right FROM channels WHERE channels.status < 2 GROUP BY node2_public_key) c2 ON c2.node2_public_key = nodes.public_key`;
-      const [nodes]: any = await DB.query(query);
-
-      for (const node of nodes) {
-        await DB.query(
-          `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`,
-          [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
-            node.channels_count_left + node.channels_count_right]);
-      }
-      logger.info('Daily node stats has updated.');
-    } catch (e) {
-      logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // We only run this on first launch
-  private async $populateHistoricalStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical stats population...`);
-
-      const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`);
-      const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`);
-
-      const date: Date = new Date(this.hardCodedStartTime);
-      const currentDate = new Date();
-      this.setDateMidnight(currentDate);
-
-      while (date < currentDate) {
-        let totalCapacity = 0;
-        let channelsCount = 0;
-
-        for (const channel of channels) {
-          if (new Date(channel.created) > date) {
-            break;
-          }
-          if (channel.closing_date === null || new Date(channel.closing_date) > date) {
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-        }
-
-        let nodeCount = 0;
-        let clearnetNodes = 0;
-        let torNodes = 0;
-        let unannouncedNodes = 0;
-
-        for (const node of nodes) {
-          if (new Date(node.first_seen) > date) {
-            break;
-          }
-          nodeCount++;
-
-          const sockets = node.sockets.split(',');
-          let isUnnanounced = true;
-          for (const socket of sockets) {
-            const hasOnion = socket.indexOf('.onion') !== -1;
-            if (hasOnion) {
-              torNodes++;
-              isUnnanounced = false;
-            }
-            const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':'))));
-            if (hasClearnet) {
-              clearnetNodes++;
-              isUnnanounced = false;
-            }
-          }
-          if (isUnnanounced) {
-            unannouncedNodes++;
-          }
-        }
-
-        const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes
-        )
-        VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`;
-
-        await DB.query(query, [
-          date.getTime() / 1000,
-          channelsCount,
-          nodeCount,
-          totalCapacity,
-          torNodes,
-          clearnetNodes,
-          unannouncedNodes,
-        ]);
-
-        date.setUTCDate(date.getUTCDate() + 1);
-      }
-
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $populateHistoricalNodeStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical node stats population...`);
-
-      const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`);
-
-      for (const node of nodes) {
-        const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]);
-        
-        const date: Date = new Date(this.hardCodedStartTime);
-        const currentDate = new Date();
-        this.setDateMidnight(currentDate);
-
-        let lastTotalCapacity = 0;
-        let lastChannelsCount = 0;
-
-        while (date < currentDate) {
-          let totalCapacity = 0;
-          let channelsCount = 0;
-          for (const channel of channels) {
-            if (new Date(channel.created) > date) {
-              break;
-            }
-            if (channel.closing_date !== null && new Date(channel.closing_date) < date) {
-              date.setUTCDate(date.getUTCDate() + 1);
-              continue;
-            }
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-
-          if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) {
-            date.setUTCDate(date.getUTCDate() + 1);
-            continue;
-          }
-
-          lastTotalCapacity = totalCapacity;
-          lastChannelsCount = channelsCount;
-  
-          const query = `INSERT INTO node_stats(
-            public_key,
-            added,
-            capacity,
-            channels
-          )
-          VALUES (?, FROM_UNIXTIME(?), ?, ?)`;
-
-          await DB.query(query, [
-            node.public_key,
-            date.getTime() / 1000,
-            totalCapacity,
-            channelsCount,
-          ]);
-          date.setUTCDate(date.getUTCDate() + 1);
-        }
-        logger.debug('Updated node_stats for: ' + node.alias);
-      }
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e));
-    }
+  /**
+   * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
+   */
+  private async $logStatsDaily(): Promise<void> {
+    const date = new Date();
+    Common.setDateMidnight(date);
+    const networkGraph = await lightningApi.$getNetworkGraph();
+    await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
+    
+    logger.info(`Updated latest network stats`);
   }
 }
 

backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts

@@ -0,0 +1,116 @@
+import { existsSync, promises } from 'fs';
+import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
+import { Common } from '../../../api/common';
+import config from '../../../config';
+import logger from '../../../logger';
+
+const fsPromises = promises;
+
+const BLOCKS_CACHE_MAX_SIZE = 100;  
+const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json';
+
+class FundingTxFetcher {
+  private running = false;
+  private blocksCache = {};
+  private channelNewlyProcessed = 0;
+  public fundingTxCache = {};
+
+  async $init(): Promise<void> {
+    // Load funding tx disk cache
+    if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) {
+      try {
+        this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
+      } catch (e) {
+        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`);
+        this.fundingTxCache = {};
+      }
+      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`);
+    }
+  }
+
+  async $fetchChannelsFundingTxs(channelIds: string[]): Promise<void> {
+    if (this.running) {
+      return;
+    }
+    this.running = true;
+    
+    const globalTimer = new Date().getTime() / 1000;
+    let cacheTimer = new Date().getTime() / 1000;
+    let loggerTimer = new Date().getTime() / 1000;
+    let channelProcessed = 0;
+    this.channelNewlyProcessed = 0;
+    for (const channelId of channelIds) {
+      await this.$fetchChannelOpenTx(channelId);
+      ++channelProcessed;
+
+      let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+      if (elapsedSeconds > 10) {
+        elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
+        logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
+          `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
+          `elapsed: ${elapsedSeconds} seconds`
+        );
+        loggerTimer = new Date().getTime() / 1000;
+      }
+
+      elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
+      if (elapsedSeconds > 60) {
+        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+        fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+        cacheTimer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (this.channelNewlyProcessed > 0) {
+      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`);
+      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+      fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+    }
+
+    this.running = false;
+  }
+  
+  public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
+    channelId = Common.channelIntegerIdToShortId(channelId);
+
+    if (this.fundingTxCache[channelId]) {
+      return this.fundingTxCache[channelId];
+    }
+
+    const parts = channelId.split('x');
+    const blockHeight = parts[0];
+    const txIdx = parts[1];
+    const outputIdx = parts[2];
+
+    let block = this.blocksCache[blockHeight];
+    // Fetch it from core
+    if (!block) {
+      const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10));
+      block = await bitcoinClient.getBlock(blockHash, 1);
+    }
+    this.blocksCache[block.height] = block;
+
+    const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse();
+    if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) {
+      for (let i = 0; i < 10; ++i) {
+        delete this.blocksCache[blocksCacheHashes[i]];
+      }
+    }
+
+    const txid = block.tx[txIdx];
+    const rawTx = await bitcoinClient.getRawTransaction(txid);
+    const tx = await bitcoinClient.decodeRawTransaction(rawTx);
+
+    this.fundingTxCache[channelId] = {
+      timestamp: block.time,
+      txid: txid,
+      value: tx.vout[outputIdx].value,
+    };
+
+    ++this.channelNewlyProcessed;
+
+    return this.fundingTxCache[channelId];
+  }
+}
+
+export default new FundingTxFetcher;

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -1,70 +1,164 @@
 import * as net from 'net';
-import maxmind, { CityResponse, AsnResponse } from 'maxmind';
+import maxmind, { CityResponse, AsnResponse, IspResponse } from 'maxmind';
 import nodesApi from '../../../api/explorer/nodes.api';
 import config from '../../../config';
 import DB from '../../../database';
 import logger from '../../../logger';
+import { ResultSetHeader } from 'mysql2';
+import * as IPCheck from '../../../utils/ipcheck.js';
 
 export async function $lookupNodeLocation(): Promise<void> {
-  logger.info(`Running node location updater using Maxmind...`);
+  let loggerTimer = new Date().getTime() / 1000;
+  let progress = 0;
+  let nodesUpdated = 0;
+  let geoNamesInserted = 0;
+
+  logger.info(`Running node location updater using Maxmind`);
   try {
     const nodes = await nodesApi.$getAllNodes();
     const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
     const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
+    const lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
 
     for (const node of nodes) {
       const sockets: string[] = node.sockets.split(',');
       for (const socket of sockets) {
         const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
         const hasClearnet = [4, 6].includes(net.isIP(ip));
+
         if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
           const city = lookupCity.get(ip);
           const asn = lookupAsn.get(ip);
-          if (city && asn) {
-            const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`;
-            const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key];
-            await DB.query(query, params);
+          const isp = lookupIsp.get(ip);
 
-             // Store Continent
-             if (city.continent?.geoname_id) {
-               await DB.query(
+          let asOverwrite: any | undefined;
+          if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
+            asOverwrite = {
+              asn: 394745,
+              name: 'Lunanode',
+            };
+          }
+          else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) {
+            asOverwrite = {
+              asn: 30058,
+              name: 'FDCservers.net',
+            };
+          }
+          else if (asn && asn.autonomous_system_number === 174) {
+            asOverwrite = {
+              asn: 174,
+              name: 'Cogent Communications',
+            };
+          }
+
+          if (city && (asn || isp)) {
+            const query = `
+              UPDATE nodes SET 
+                as_number = ?, 
+                city_id = ?, 
+                country_id = ?, 
+                subdivision_id = ?, 
+                longitude = ?, 
+                latitude = ?, 
+                accuracy_radius = ?
+              WHERE public_key = ?
+            `;
+
+            const params = [
+              asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+              city.city?.geoname_id,
+              city.country?.geoname_id,
+              city.subdivisions ? city.subdivisions[0].geoname_id : null,
+              city.location?.longitude,
+              city.location?.latitude,
+              city.location?.accuracy_radius,
+              node.public_key
+            ];
+            let result = await DB.query<ResultSetHeader>(query, params);
+            if (result[0].changedRows ?? 0 > 0) {
+              ++nodesUpdated;
+            }
+
+            // Store Continent
+            if (city.continent?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
                 [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
-             }
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
 
-             // Store Country
-             if (city.country?.geoname_id) {
-               await DB.query(
+            // Store Country
+            if (city.country?.geoname_id) {
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
                 [city.country?.geoname_id, JSON.stringify(city.country?.names)]);
-             }
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
+
+            // Store Country ISO code
+            if (city.country?.iso_code) {
+              result = await DB.query<ResultSetHeader>(
+                `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
+                [city.country?.geoname_id, city.country?.iso_code]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
+            }
 
             // Store Division
             if (city.subdivisions && city.subdivisions[0]) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
                 [city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store City
             if (city.city?.geoname_id) {
-              await DB.query(
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
                 [city.city?.geoname_id, JSON.stringify(city.city?.names)]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
 
             // Store AS name
-            if (asn.autonomous_system_organization) {
-              await DB.query(
+            if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
+              result = await DB.query<ResultSetHeader>(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
-                [asn.autonomous_system_number, JSON.stringify(asn.autonomous_system_organization)]);
+                [
+                  asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+                  JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
+                ]);
+              if (result[0].changedRows ?? 0 > 0) {
+                ++geoNamesInserted;
+              }
             }
           }
+
+          ++progress;
+          const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+          if (elapsedSeconds > 10) {
+            logger.info(`Updating node location data ${progress}/${nodes.length}`);
+            loggerTimer = new Date().getTime() / 1000;
+          }
         }
       }
     }
-    logger.info(`Node location data updated.`);
+
+    if (nodesUpdated > 0) {
+      logger.info(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+    } else {
+      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+    }
   } catch (e) {
     logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
   }
-}
+}

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -0,0 +1,542 @@
+import DB from '../../../database';
+import { promises } from 'fs';
+import logger from '../../../logger';
+import fundingTxFetcher from './funding-tx-fetcher';
+import config from '../../../config';
+import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
+import { isIP } from 'net';
+import { Common } from '../../../api/common';
+import channelsApi from '../../../api/explorer/channels.api';
+import nodesApi from '../../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+
+const fsPromises = promises;
+
+class LightningStatsImporter {
+  topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
+
+  async $run(): Promise<void> {
+    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+    logger.info('Caching funding txs for currently existing channels');
+    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+
+    if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
+      return;
+    }
+
+    await this.$importHistoricalLightningStats();
+    await this.$cleanupIncorrectSnapshot();
+  }
+
+  /**
+   * Generate LN network stats for one day
+   */
+  public async computeNetworkStats(timestamp: number,
+    networkGraph: ILightningApi.NetworkGraph, isHistorical: boolean = false): Promise<unknown> {
+    // Node counts and network shares
+    let clearnetNodes = 0;
+    let torNodes = 0;
+    let clearnetTorNodes = 0;
+    let unannouncedNodes = 0;
+
+    const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`);
+    const nodesInDb = {};
+    for (const node of nodesInDbRaw) {
+      nodesInDb[node.public_key] = node;
+    }
+
+    for (const node of networkGraph.nodes) {
+      // If we don't know about this node, insert it in db
+      if (isHistorical === true && !nodesInDb[node.pub_key]) {
+        await nodesApi.$saveNode({
+          last_update: node.last_update,
+          pub_key: node.pub_key,
+          alias: node.alias,
+          addresses: node.addresses,
+          color: node.color,
+          features: node.features,
+        });
+        nodesInDb[node.pub_key] = node;
+      } else {
+        await nodesApi.$updateNodeSockets(node.pub_key, node.addresses);
+      }
+
+      let hasOnion = false;
+      let hasClearnet = false;
+      let isUnnanounced = true;
+
+      for (const socket of (node.addresses ?? [])) {
+        if (!socket.network?.length && !socket.addr?.length) {
+          continue;
+        }
+        hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1 || socket.addr.indexOf('torv2') !== -1 || socket.addr.indexOf('torv3') !== -1;
+        hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])) || socket.addr.indexOf('ipv4') !== -1 || socket.addr.indexOf('ipv6') !== -1;;
+      }
+      if (hasOnion && hasClearnet) {
+        clearnetTorNodes++;
+        isUnnanounced = false;
+      } else if (hasOnion) {
+        torNodes++;
+        isUnnanounced = false;
+      } else if (hasClearnet) {
+        clearnetNodes++;
+        isUnnanounced = false;
+      }
+      if (isUnnanounced) {
+        unannouncedNodes++;
+      }
+    }
+
+    // Channels and node historical stats
+    const nodeStats = {};
+    let capacity = 0;
+    let avgFeeRate = 0;
+    let avgBaseFee = 0;
+    const capacities: number[] = [];
+    const feeRates: number[] = [];
+    const baseFees: number[] = [];
+    const alreadyCountedChannels = {};
+    
+    const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
+    const channelsInDb = {};
+    for (const channel of channelsInDbRaw) {
+      channelsInDb[channel.short_id] = channel;
+    }
+
+    for (const channel of networkGraph.edges) {
+      const short_id = Common.channelIntegerIdToShortId(channel.channel_id);
+
+      const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
+      if (!tx) {
+        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`);
+        continue;
+      }
+
+      // If we don't know about this channel, insert it in db
+      if (isHistorical === true && !channelsInDb[short_id]) {
+        await channelsApi.$saveChannel({
+          channel_id: short_id,
+          chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
+          last_update: channel.last_update,
+          node1_pub: channel.node1_pub,
+          node2_pub: channel.node2_pub,
+          capacity: (tx.value * 100000000).toString(),
+          node1_policy: null,
+          node2_policy: null,
+        }, 0);
+        channelsInDb[channel.channel_id] = channel;
+      }
+
+      if (!nodeStats[channel.node1_pub]) {
+        nodeStats[channel.node1_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      if (!nodeStats[channel.node2_pub]) {
+        nodeStats[channel.node2_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      
+      if (!alreadyCountedChannels[short_id]) {
+        capacity += Math.round(tx.value * 100000000);
+        capacities.push(Math.round(tx.value * 100000000));
+        alreadyCountedChannels[short_id] = true;
+
+        nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node1_pub].channels++;
+        nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node2_pub].channels++;
+      }
+
+      if (isHistorical === false) { // Coming from the node
+        for (const policy of [channel.node1_policy, channel.node2_policy]) {
+          if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) {
+            avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
+            feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
+          }  
+          if (policy && parseInt(policy.fee_base_msat, 10) < 5000) {
+            avgBaseFee += parseInt(policy.fee_base_msat, 10);
+            baseFees.push(parseInt(policy.fee_base_msat, 10));
+          }
+        }
+      } else {
+        // @ts-ignore
+        if (channel.node1_policy.fee_rate_milli_msat < 5000) {
+          // @ts-ignore
+          avgFeeRate += parseInt(channel.node1_policy.fee_rate_milli_msat, 10);
+          // @ts-ignore
+          feeRates.push(parseInt(channel.node1_policy.fee_rate_milli_msat), 10);
+        }
+        // @ts-ignore
+        if (channel.node1_policy.fee_base_msat < 5000) {
+          // @ts-ignore
+          avgBaseFee += parseInt(channel.node1_policy.fee_base_msat, 10);
+          // @ts-ignore
+          baseFees.push(parseInt(channel.node1_policy.fee_base_msat), 10);
+        }
+      }
+    }
+
+    let medCapacity = 0;
+    let medFeeRate = 0;
+    let medBaseFee = 0;
+    let avgCapacity = 0;
+
+    avgFeeRate /= Math.max(networkGraph.edges.length, 1);
+    avgBaseFee /= Math.max(networkGraph.edges.length, 1);
+
+    if (capacities.length > 0) {
+      medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
+      avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
+    }
+    if (feeRates.length > 0) {
+      medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
+    }
+    if (baseFees.length > 0) {
+      medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
+    }
+
+    let query = `INSERT INTO lightning_stats(
+        added,
+        channel_count,
+        node_count,
+        total_capacity,
+        tor_nodes,
+        clearnet_nodes,
+        unannounced_nodes,
+        clearnet_tor_nodes,
+        avg_capacity,
+        avg_fee_rate,
+        avg_base_fee_mtokens,
+        med_capacity,
+        med_fee_rate,
+        med_base_fee_mtokens
+      )
+      VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+      added = FROM_UNIXTIME(?),
+      channel_count = ?,
+      node_count = ?,
+      total_capacity = ?,
+      tor_nodes = ?,
+      clearnet_nodes = ?,
+      unannounced_nodes = ?,
+      clearnet_tor_nodes = ?,
+      avg_capacity = ?,
+      avg_fee_rate = ?,
+      avg_base_fee_mtokens = ?,
+      med_capacity = ?,
+      med_fee_rate = ?,
+      med_base_fee_mtokens = ?
+    `;
+
+    await DB.query(query, [
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+    ]);
+
+    for (const public_key of Object.keys(nodeStats)) {
+      query = `INSERT INTO node_stats(
+          public_key,
+          added,
+          capacity,
+          channels
+        )
+        VALUES (?, FROM_UNIXTIME(?), ?, ?)
+        ON DUPLICATE KEY UPDATE
+        added = FROM_UNIXTIME(?),
+        capacity = ?,
+        channels = ?
+      `;
+
+      await DB.query(query, [
+        public_key,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+      ]);
+
+      if (!isHistorical) {
+        await DB.query(
+          `UPDATE nodes SET capacity = ?, channels = ? WHERE public_key = ?`,
+          [
+            nodeStats[public_key].capacity,
+            nodeStats[public_key].channels,
+            public_key,
+          ]
+        );
+      }
+    }
+
+    return {
+      added: timestamp,
+      node_count: networkGraph.nodes.length
+    };
+  }
+
+  /**
+   * Import topology files LN historical data into the database
+   */
+  async $importHistoricalLightningStats(): Promise<void> {
+    logger.debug('Run the historical importer');
+    try {
+      let fileList: string[] = [];
+      try {
+        fileList = await fsPromises.readdir(this.topologiesFolder);
+      } catch (e) {
+        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`);
+        throw e;
+      }
+      // Insert history from the most recent to the oldest
+      // This also put the .json cached files first
+      fileList.sort().reverse();
+
+      const [rows]: any[] = await DB.query(`
+        SELECT UNIX_TIMESTAMP(added) AS added
+        FROM lightning_stats
+        ORDER BY added DESC
+      `);
+      const existingStatsTimestamps = {};
+      for (const row of rows) {
+        existingStatsTimestamps[row.added] = row;
+      }
+
+      // For logging purpose
+      let processed = 10;
+      let totalProcessed = 0;
+      let logStarted = false;
+
+      for (const filename of fileList) {
+        processed++;
+
+        const timestamp = parseInt(filename.split('_')[1], 10);
+
+        // Stats exist already, don't calculate/insert them
+        if (existingStatsTimestamps[timestamp] !== undefined) {
+          totalProcessed++;
+          continue;
+        }
+
+        if (filename.indexOf('topology_') === -1) {
+          totalProcessed++;
+          continue;
+        }
+
+        logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
+        let fileContent = '';
+        try {
+          fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
+        } catch (e: any) {
+          if (e.errno == -1) { // EISDIR - Ignore directorie
+            totalProcessed++;
+            continue;
+          }
+          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`);
+          totalProcessed++;
+          continue;
+        }
+
+        let graph;
+        try {
+          graph = JSON.parse(fileContent);
+          graph = await this.cleanupTopology(graph);
+        } catch (e) {
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
+          totalProcessed++;
+          continue;
+        }
+    
+        if (this.isIncorrectSnapshot(timestamp, graph)) {
+          logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
+          ++totalProcessed;
+          continue;
+        }
+
+        if (!logStarted) {
+          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
+          logStarted = true;
+        }
+        
+        const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
+        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
+
+        totalProcessed++;
+
+        if (processed > 10) {
+          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+          processed = 0;
+        } else {
+          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+        }
+        await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
+        const stat = await this.computeNetworkStats(timestamp, graph, true);
+
+        existingStatsTimestamps[timestamp] = stat;
+      }
+
+      if (totalProcessed > 0) {
+        logger.info(`Lightning network stats historical import completed`);
+      }
+    } catch (e) {
+      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  cleanupTopology(graph): ILightningApi.NetworkGraph {
+    const newGraph = {
+      nodes: <ILightningApi.Node[]>[],
+      edges: <ILightningApi.Channel[]>[],
+    };
+
+    for (const node of graph.nodes) {
+      const addressesParts = (node.addresses ?? '').split(',');
+      const addresses: any[] = [];
+      for (const address of addressesParts) {
+        const formatted = Common.findSocketNetwork(address);
+        addresses.push({
+          network: formatted.network,
+          addr: formatted.url
+        });
+      }
+
+      let rgb = node.rgb_color ?? '#000000';
+      if (rgb.indexOf('#') === -1) {
+        rgb = `#${rgb}`;
+      }
+      newGraph.nodes.push({
+        last_update: node.timestamp ?? 0,
+        pub_key: node.id ?? null,
+        alias: node.alias ?? node.id.slice(0, 20),
+        addresses: addresses,
+        color: rgb,
+        features: {},
+      });
+    }
+
+    for (const adjacency of graph.adjacency) {
+      if (adjacency.length === 0) {
+        continue;
+      } else {
+        for (const edge of adjacency) {
+          newGraph.edges.push({
+            channel_id: edge.scid,
+            chan_point: '',
+            last_update: edge.timestamp,
+            node1_pub: edge.source ?? null,
+            node2_pub: edge.destination ?? null,
+            capacity: '0', // Will be fetch later
+            node1_policy: {
+              time_lock_delta: edge.cltv_expiry_delta,
+              min_htlc: edge.htlc_minimim_msat,
+              fee_base_msat: edge.fee_base_msat,
+              fee_rate_milli_msat: edge.fee_proportional_millionths,
+              max_htlc_msat: edge.htlc_maximum_msat,
+              last_update: edge.timestamp,
+              disabled: false,          
+            },
+            node2_policy: null,
+          });
+        }
+      }
+    }
+
+    return newGraph;
+  }
+
+  private isIncorrectSnapshot(timestamp, graph): boolean {
+    if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
+        return true;
+    }
+    if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
+      return true;
+    }
+    if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
+      return true;
+    }
+    if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
+      return true;
+    }
+    if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
+      return true;
+    }
+    if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private async $cleanupIncorrectSnapshot(): Promise<void> {
+    // We do not run this one automatically because those stats are not supposed to be inserted in the first
+    // place, but I write them here to remind us we manually run those queries
+
+    // DELETE FROM lightning_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
+    // )
+
+    // DELETE FROM node_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 
+    // )
+  }
+}
+
+export default new LightningStatsImporter;

backend/src/tasks/pools-updater.ts

@@ -12,14 +12,11 @@ import * as https from 'https';
  */
 class PoolsUpdater {
   lastRun: number = 0;
-  currentSha: any = undefined;
-  poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
-  treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
+  currentSha: string | undefined = undefined;
+  poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
+  treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
-  constructor() {
-  }
-
-  public async updatePoolsJson() {
+  public async updatePoolsJson(): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -77,7 +74,7 @@ class PoolsUpdater {
   /**
    * Fetch our latest pools.json sha from the db
    */
-  private async updateDBSha(githubSha: string) {
+  private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
     if (config.DATABASE.ENABLED === true) {
       try {

backend/src/tasks/price-feeds/kraken-api.ts

@@ -62,7 +62,7 @@ class KrakenApi implements PriceFeed {
     // CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019)
     // AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020)
 
-    const priceHistory: any = {}; // map: timestamp -> Prices
+    let priceHistory: any = {}; // map: timestamp -> Prices
 
     for (const currency of this.currencies) {
       const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency);
@@ -83,6 +83,10 @@ class KrakenApi implements PriceFeed {
     }
 
     for (const time in priceHistory) {
+      if (priceHistory[time].USD === -1) {
+        delete priceHistory[time];
+        continue;
+      }
       await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]);
     }
 

backend/src/tasks/price-updater.ts

@@ -1,4 +1,6 @@
 import * as fs from 'fs';
+import path from "path";
+import { Common } from '../api/common';
 import config from '../config';
 import logger from '../logger';
 import PricesRepository from '../repositories/PricesRepository';
@@ -34,10 +36,10 @@ export interface Prices {
 }
 
 class PriceUpdater {
-  historyInserted: boolean = false;
-  lastRun: number = 0;
-  lastHistoricalRun: number = 0;
-  running: boolean = false;
+  public historyInserted = false;
+  lastRun = 0;
+  lastHistoricalRun = 0;
+  running = false;
   feeds: PriceFeed[] = [];
   currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
   latestPrices: Prices;
@@ -158,7 +160,7 @@ class PriceUpdater {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
     // Insert MtGox weekly prices
-    const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
+    const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
     const prices = this.getEmptyPricesObj();
     let insertedCount: number = 0;
     for (const price of pricesJson) {

backend/src/utils/ipcheck.js

@@ -0,0 +1,119 @@
+var net = require('net');
+
+var IPCheck = module.exports = function(input) {
+  var self = this;
+
+  if (!(self instanceof IPCheck)) {
+    return new IPCheck(input);
+  }
+
+  self.input = input;
+  self.parse();
+};
+
+IPCheck.prototype.parse = function() {
+  var self = this;
+
+  if (!self.input || typeof self.input !== 'string') return self.valid = false;
+
+  var ip;
+
+  var pos = self.input.lastIndexOf('/');
+  if (pos !== -1) {
+    ip = self.input.substring(0, pos);
+    self.mask = +self.input.substring(pos + 1);
+  } else {
+    ip = self.input;
+    self.mask = null;
+  }
+
+  self.ipv = net.isIP(ip);
+  self.valid = !!self.ipv && !isNaN(self.mask);
+
+  if (!self.valid) return;
+
+  // default mask = 32 for ipv4 and 128 for ipv6
+  if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
+
+  if (self.ipv === 4) {
+    // difference between ipv4 and ipv6 masks
+    self.mask += 96;
+  }
+
+  if (self.mask < 0 || self.mask > 128) {
+    self.valid = false;
+    return;
+  }
+
+  self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
+
+  if(self.ipv === 4){
+    self.parseIPv4(ip);
+  }else{
+    self.parseIPv6(ip);
+  }
+};
+
+IPCheck.prototype.parseIPv4 = function(ip) {
+  var self = this;
+
+  // ipv4 addresses live under ::ffff:0:0
+  self.address[10] = self.address[11] = 0xff;
+
+  var octets = ip.split('.');
+  for (var i = 0; i < 4; i++) {
+    self.address[i + 12] = parseInt(octets[i], 10);
+  }
+};
+
+
+var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
+
+IPCheck.prototype.parseIPv6 = function(ip) {
+  var self = this;
+
+  var transitionalMatch = V6_TRANSITIONAL.exec(ip);
+  if(transitionalMatch){
+    self.parseIPv4(transitionalMatch[1]);
+    return;
+  }
+
+  var bits = ip.split(':');
+  if (bits.length < 8) {
+    ip = ip.replace('::', Array(11 - bits.length).join(':'));
+    bits = ip.split(':');
+  }
+
+  var j = 0;
+  for (var i = 0; i < bits.length; i += 1) {
+    var x = bits[i] ? parseInt(bits[i], 16) : 0;
+    self.address[j++] = x >> 8;
+    self.address[j++] = x & 0xff;
+  }
+};
+
+IPCheck.prototype.match = function(cidr) {
+  var self = this;
+
+  if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
+  if (!self.valid || !cidr.valid) return false;
+
+  var mask = cidr.mask;
+  var i = 0;
+
+  while (mask >= 8) {
+    if (self.address[i] !== cidr.address[i]) return false;
+
+    i++;
+    mask -= 8;
+  }
+
+  var shift = 8 - mask;
+  return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
+};
+
+
+IPCheck.match = function(ip, cidr) {
+  ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
+  return ip.match(cidr);
+};

backend/testSetup.ts

@@ -0,0 +1,5 @@
+jest.mock('./mempool-config.json', () => ({}), { virtual: true });
+jest.mock('./src/logger.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true });

backend/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig",
+  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"],
+  "compilerOptions": {
+    "types": ["node"]
+  },
+
+}

backend/tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
-    "types": ["node"],
     "module": "commonjs",
     "target": "esnext",
+    "types": ["node", "jest"],
     "lib": ["es2019", "dom"],
     "strict": true,
     "noImplicitAny": false,
@@ -13,7 +13,8 @@
       "node_modules/@types"
     ],
     "allowSyntheticDefaultImports": true,
-    "esModuleInterop": true
+    "esModuleInterop": true,
+    "allowJs": true,
   },
   "include": [
     "src/**/*.ts"
@@ -21,4 +22,4 @@
   "exclude": [
     "dist/**"
   ]
-}
+}

contributors/WesVleuten.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of September 1, 2022.
+
+Signed: WesVleuten

contributors/junderw.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 19, 2022.
+
+Signed: junderw

contributors/oleonardolima.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 25, 2022.
+
+Signed: oleonardolima

docker/README.md

@@ -102,7 +102,9 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
-    "STDOUT_LOG_MIN_PRIORITY": "info"
+    "STDOUT_LOG_MIN_PRIORITY": "info",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
   },
 ```
 
@@ -126,6 +128,8 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
+      MEMPOOL_POOLS_JSON_URL: ""
+      MEMPOOL_POOLS_JSON_TREE_URL: ""
       ...
 ```
 
@@ -346,3 +350,68 @@ Corresponding `docker-compose.yml` overrides:
       PRICE_DATA_SERVER_CLEARNET_URL: ""
       ...
 ```
+
+<br/>
+
+`mempool-config.json`:
+```
+  "LIGHTNING": {
+    "ENABLED": false
+    "BACKEND": "lnd"
+    "TOPOLOGY_FOLDER": ""
+    "STATS_REFRESH_INTERVAL": 600
+    "GRAPH_REFRESH_INTERVAL": 600
+    "LOGGER_UPDATE_INTERVAL": 30
+  }
+```
+
+Corresponding `docker-compose.yml` overrides:
+```
+  api:
+    environment:
+      LIGHTNING_ENABLED: false
+      LIGHTNING_BACKEND: "lnd"
+      LIGHTNING_TOPOLOGY_FOLDER: ""
+      LIGHTNING_STATS_REFRESH_INTERVAL: 600
+      LIGHTNING_GRAPH_REFRESH_INTERVAL: 600
+      LIGHTNING_LOGGER_UPDATE_INTERVAL: 30
+      ...
+```
+
+<br/>
+
+`mempool-config.json`:
+```
+  "LND": {
+    "TLS_CERT_PATH": ""
+    "MACAROON_PATH": ""
+    "REST_API_URL": "https://localhost:8080"
+  }
+```
+
+Corresponding `docker-compose.yml` overrides:
+```
+  api:
+    environment:
+      LND_TLS_CERT_PATH: ""
+      LND_MACAROON_PATH: ""
+      LND_REST_API_URL: "https://localhost:8080"
+      ...
+```
+
+<br/>
+
+`mempool-config.json`:
+```
+  "CLIGHTNING": {
+    "SOCKET": ""
+  }
+```
+
+Corresponding `docker-compose.yml` overrides:
+```
+  api:
+    environment:
+      CLIGHTNING_SOCKET: ""
+      ...
+```

docker/backend/Dockerfile

@@ -1,7 +1,7 @@
 FROM node:16.16.0-buster-slim AS builder
 
 ARG commitHash
-ENV DOCKER_COMMIT_HASH=${commitHash}
+ENV MEMPOOL_COMMIT_HASH=${commitHash}
 
 WORKDIR /build
 COPY . .
@@ -9,18 +9,15 @@ COPY . .
 RUN apt-get update
 RUN apt-get install -y build-essential python3 pkg-config
 RUN npm install --omit=dev --omit=optional
-RUN npm run build
+RUN npm run package
 
 FROM node:16.16.0-buster-slim
 
 WORKDIR /backend
 
-COPY --from=builder /build/ .
-
-RUN chmod +x /backend/start.sh
-RUN chmod +x /backend/wait-for-it.sh
-
-RUN chown -R 1000:1000 /backend && chmod -R 755 /backend
+RUN chown 1000:1000 ./
+COPY --from=builder --chown=1000:1000 /build/package ./package/
+COPY --from=builder --chown=1000:1000 /build/mempool-config.json /build/start.sh /build/wait-for-it.sh ./
 
 USER 1000
 

docker/backend/mempool-config.json

@@ -67,6 +67,22 @@
     "ENABLED": __BISQ_ENABLED__,
     "DATA_PATH": "__BISQ_DATA_PATH__"
   },
+  "LIGHTNING": {
+    "ENABLED": __LIGHTNING_ENABLED__,
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "STATS_REFRESH_INTERVAL": __LIGHTNING_STATS_REFRESH_INTERVAL__,
+    "GRAPH_REFRESH_INTERVAL": __LIGHTNING_GRAPH_REFRESH_INTERVAL__,
+    "LOGGER_UPDATE_INTERVAL": __LIGHTNING_LOGGER_UPDATE_INTERVAL__,
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__"
+  },
+  "LND": {
+    "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__",
+    "MACAROON_PATH": "__LND_MACAROON_PATH__",
+    "REST_API_URL": "__LND_REST_API_URL__"
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  },
   "SOCKS5PROXY": {
     "ENABLED": __SOCKS5PROXY_ENABLED__,
     "USE_ONION": __SOCKS5PROXY_USE_ONION__,

docker/backend/start.sh

@@ -24,6 +24,8 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
+__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
+__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -89,6 +91,22 @@ __EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http:
 __EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api}
 __EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api}
 
+# LIGHTNING
+__LIGHTNING_ENABLED__=${LIGHTNING_ENABLED:=false}
+__LIGHTNING_BACKEND__=${LIGHTNING_BACKEND:="lnd"}
+__LIGHTNING_TOPOLOGY_FOLDER__=${LIGHTNING_TOPOLOGY_FOLDER:=""}
+__LIGHTNING_STATS_REFRESH_INTERVAL__=${LIGHTNING_STATS_REFRESH_INTERVAL:=600}
+__LIGHTNING_GRAPH_REFRESH_INTERVAL__=${LIGHTNING_GRAPH_REFRESH_INTERVAL:=600}
+__LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30}
+
+# LND
+__LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""}
+__LND_MACAROON_PATH__=${LND_MACAROON_PATH:=""}
+__LND_REST_API_URL__=${LND_REST_API_URL:="https://localhost:8080"}
+
+# CLN
+__CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}
+
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@@ -114,6 +132,8 @@ sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.jso
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
+sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
@@ -169,4 +189,20 @@ sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_ONION__!${__EXTERNAL_DATA_SERVER_LIQUID_
 sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_URL__!${__EXTERNAL_DATA_SERVER_BISQ_URL__}!g" mempool-config.json
 sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_ONION__!${__EXTERNAL_DATA_SERVER_BISQ_ONION__}!g" mempool-config.json
 
-node /backend/dist/index.js
+# LIGHTNING
+sed -i "s!__LIGHTNING_ENABLED__!${__LIGHTNING_ENABLED__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_BACKEND__!${__LIGHTNING_BACKEND__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_TOPOLOGY_FOLDER__!${__LIGHTNING_TOPOLOGY_FOLDER__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_STATS_REFRESH_INTERVAL__!${__LIGHTNING_STATS_REFRESH_INTERVAL__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_GRAPH_REFRESH_INTERVAL__!${__LIGHTNING_GRAPH_REFRESH_INTERVAL__}!g" mempool-config.json
+sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTERVAL__}!g" mempool-config.json
+
+# LND
+sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json
+sed -i "s!__LND_MACAROON_PATH__!${__LND_MACAROON_PATH__}!g" mempool-config.json
+sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
+
+# CLN
+sed -i "s!__CLIGHTNING_SOCKET__!${__CLIGHTNING_SOCKET__}!g" mempool-config.json
+
+node /backend/package/index.js

docker/init.sh

@@ -1,10 +1,7 @@
 #!/bin/sh
 
 #backend
-gitMaster="\.\.\/\.git\/refs\/heads\/master"
-git ls-remote https://github.com/mempool/mempool.git "$1^{}" | awk '{ print $1}' > ./backend/master
 cp ./docker/backend/* ./backend/
-sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts
 
 #frontend
 localhostIP="127.0.0.1"

frontend/.eslintrc

@@ -14,10 +14,11 @@
     "@typescript-eslint/ban-types": 1,
     "@typescript-eslint/no-empty-function": 1,
     "@typescript-eslint/no-explicit-any": 1,
-    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-inferrable-types": 0,
     "@typescript-eslint/no-namespace": 1,
     "@typescript-eslint/no-this-alias": 1,
     "@typescript-eslint/no-var-requires": 1,
+    "@typescript-eslint/explicit-function-return-type": 1,
     "no-case-declarations": 1,
     "no-console": 1,
     "no-constant-condition": 1,
@@ -29,6 +30,9 @@
     "no-useless-catch": 1,
     "no-var": 1,
     "prefer-const": 1,
-    "prefer-rest-params": 1
+    "prefer-rest-params": 1,
+    "quotes": [1, "single", { "allowTemplateLiterals": true }],
+    "semi": 1,
+    "eqeqeq": 1
   }
 }

frontend/angular.json

@@ -170,6 +170,10 @@
           },
           "configurations": {
             "production": {
+              "assets": [
+                "src/favicon.ico",
+                "src/robots.txt"
+              ],
               "fileReplacements": [
                 {
                   "replace": "src/environments/environment.ts",

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -35,21 +35,23 @@ const getRectangle = ($el) => $el[0].getBoundingClientRect();
 describe('Mainnet', () => {
   beforeEach(() => {
     //cy.intercept('/sockjs-node/info*').as('socket');
-    cy.intercept('/api/block-height/*').as('block-height');
-    cy.intercept('/api/block/*').as('block');
-    cy.intercept('/api/block/*/txs/0').as('block-txs');
-    cy.intercept('/api/tx/*/outspends').as('tx-outspends');
-    cy.intercept('/resources/pools.json').as('pools');
+    // cy.intercept('/api/block-height/*').as('block-height');
+    // cy.intercept('/api/v1/block/*').as('block');
+    // cy.intercept('/api/block/*/txs/0').as('block-txs');
+    // cy.intercept('/api/v1/block/*/summary').as('block-summary');
+    // cy.intercept('/api/v1/outspends/*').as('outspends');
+    // cy.intercept('/api/tx/*/outspends').as('tx-outspends');
+    // cy.intercept('/resources/pools.json').as('pools');
 
     // Search Auto Complete
     cy.intercept('/api/address-prefix/1wiz').as('search-1wiz');
     cy.intercept('/api/address-prefix/1wizS').as('search-1wizS');
     cy.intercept('/api/address-prefix/1wizSA').as('search-1wizSA');
 
-    Cypress.Commands.add('waitForBlockData', () => {
-      cy.wait('@tx-outspends');
-      cy.wait('@pools');
-    });
+    // Cypress.Commands.add('waitForBlockData', () => {
+    //   cy.wait('@tx-outspends');
+    //   cy.wait('@pools');
+    // });
   });
 
   if (baseModule === 'mempool') {
@@ -409,7 +411,7 @@ describe('Mainnet', () => {
 
     it('loads the tv screen - desktop', () => {
       cy.viewport('macbook-16');
-      cy.visit('/');
+      cy.visit('/graphs/mempool');
       cy.waitForSkeletonGone();
       cy.get('#btn-tv').click().then(() => {
         cy.viewport('macbook-16');

frontend/cypress/e2e/signet/signet.spec.ts

@@ -60,10 +60,10 @@ describe('Signet', () => {
       });
     });
 
-    describe('tv mode', () => {
+    describe.skip('tv mode', () => {
       it('loads the tv screen - desktop', () => {
         cy.viewport('macbook-16');
-        cy.visit('/signet');
+        cy.visit('/signet/graphs');
         cy.waitForSkeletonGone();
         cy.get('#btn-tv').click().then(() => {
           cy.get('.chart-holder').should('be.visible');
@@ -73,19 +73,17 @@ describe('Signet', () => {
       });
 
       it('loads the tv screen - mobile', () => {
-        cy.visit('/signet');
+        cy.visit('/signet/graphs');
         cy.waitForSkeletonGone();
         cy.get('#btn-tv').click().then(() => {
           cy.viewport('iphone-8');
           cy.get('.chart-holder').should('be.visible');
           cy.get('.tv-only').should('not.exist');
-          //TODO: Remove comment when the bug is fixed
-          //cy.get('#mempool-block-0').should('be.visible');
+          cy.get('#mempool-block-0').should('be.visible');
         });
       });
     });
 
-
     it('loads the api screen', () => {
       cy.visit('/signet');
       cy.waitForSkeletonGone();

frontend/cypress/e2e/testnet/testnet.spec.ts

@@ -63,18 +63,17 @@ describe('Testnet', () => {
     describe('tv mode', () => {
       it('loads the tv screen - desktop', () => {
         cy.viewport('macbook-16');
-        cy.visit('/testnet');
+        cy.visit('/testnet/graphs');
         cy.waitForSkeletonGone();
         cy.get('#btn-tv').click().then(() => {
           cy.wait(1000);
           cy.get('.tv-only').should('not.exist');
-          //TODO: Remove comment when the bug is fixed
-          //cy.get('#mempool-block-0').should('be.visible');
+          cy.get('#mempool-block-0').should('be.visible');
         });
       });
 
       it('loads the tv screen - mobile', () => {
-        cy.visit('/testnet');
+        cy.visit('/testnet/graphs');
         cy.waitForSkeletonGone();
         cy.get('#btn-tv').click().then(() => {
           cy.viewport('iphone-6');

frontend/package-lock.json

@@ -34,6 +34,7 @@
         "clipboard": "^2.0.10",
         "domino": "^2.1.6",
         "echarts": "~5.3.2",
+        "echarts-gl": "^2.0.9",
         "express": "^4.17.1",
         "lightweight-charts": "~3.8.0",
         "ngx-echarts": "8.0.1",
@@ -6396,6 +6397,11 @@
         "webpack": ">=4.0.1"
       }
     },
+    "node_modules/claygl": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/claygl/-/claygl-1.3.0.tgz",
+      "integrity": "sha512-+gGtJjT6SSHD2l2yC3MCubW/sCV40tZuSs5opdtn79vFSGUgp/lH139RNEQ6Jy078/L0aV8odCw8RSrUcMfLaQ=="
+    },
     "node_modules/clean-stack": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
@@ -8107,6 +8113,18 @@
         "zrender": "5.3.1"
       }
     },
+    "node_modules/echarts-gl": {
+      "version": "2.0.9",
+      "resolved": "https://registry.npmjs.org/echarts-gl/-/echarts-gl-2.0.9.tgz",
+      "integrity": "sha512-oKeMdkkkpJGWOzjgZUsF41DOh6cMsyrGGXimbjK2l6Xeq/dBQu4ShG2w2Dzrs/1bD27b2pLTGSaUzouY191gzA==",
+      "dependencies": {
+        "claygl": "^1.2.1",
+        "zrender": "^5.1.1"
+      },
+      "peerDependencies": {
+        "echarts": "^5.1.2"
+      }
+    },
     "node_modules/echarts/node_modules/tslib": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz",
@@ -22520,6 +22538,11 @@
       "integrity": "sha512-g38K9Cm5WRwlaH6g03B9OEz/0qRizI+2I7n+Gz+L5DxXJAPAiWQvwlYNm1V1jkdpUv95bOe/ASm2vfi/G560jQ==",
       "requires": {}
     },
+    "claygl": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/claygl/-/claygl-1.3.0.tgz",
+      "integrity": "sha512-+gGtJjT6SSHD2l2yC3MCubW/sCV40tZuSs5opdtn79vFSGUgp/lH139RNEQ6Jy078/L0aV8odCw8RSrUcMfLaQ=="
+    },
     "clean-stack": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
@@ -23866,6 +23889,15 @@
         }
       }
     },
+    "echarts-gl": {
+      "version": "2.0.9",
+      "resolved": "https://registry.npmjs.org/echarts-gl/-/echarts-gl-2.0.9.tgz",
+      "integrity": "sha512-oKeMdkkkpJGWOzjgZUsF41DOh6cMsyrGGXimbjK2l6Xeq/dBQu4ShG2w2Dzrs/1bD27b2pLTGSaUzouY191gzA==",
+      "requires": {
+        "claygl": "^1.2.1",
+        "zrender": "^5.1.1"
+      }
+    },
     "ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",

frontend/package.json

@@ -34,8 +34,8 @@
     "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
     "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed",
     "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
-    "sync-assets": "node sync-assets.js && rsync -av ./dist/mempool/browser/en-US/resources ./dist/mempool/browser/resources",
-    "sync-assets-dev": "node sync-assets.js dev",
+    "sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources/'",
+    "sync-assets-dev": "node sync-assets.js 'src/resources/'",
     "generate-config": "node generate-config.js",
     "build-mempool.js": "npm run build-mempool-js && npm run build-mempool-liquid-js && npm run build-mempool-bisq-js",
     "build-mempool-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index.js --standalone mempoolJS > ./dist/mempool/browser/en-US/mempool.js",
@@ -88,6 +88,7 @@
     "clipboard": "^2.0.10",
     "domino": "^2.1.6",
     "echarts": "~5.3.2",
+    "echarts-gl": "^2.0.9",
     "express": "^4.17.1",
     "lightweight-charts": "~3.8.0",
     "ngx-echarts": "8.0.1",

frontend/proxy.conf.js

@@ -85,7 +85,7 @@ if (configContent && configContent.BASE_MODULE == "liquid") {
     });
 } else {
     PROXY_CONFIG.push({
-        context: ['/resources/pools.json', '/resources/assets.json', '/resources/assets.minimal.json'],
+        context: ['/resources/pools.json', '/resources/assets.json', '/resources/assets.minimal.json', '/resources/worldmap.json'],
         target: "https://mempool.space",
         secure: false,
         changeOrigin: true,

frontend/src/app/app-routing.module.ts

@@ -1,5 +1,6 @@
 import { NgModule } from '@angular/core';
-import { Routes, RouterModule, PreloadAllModules } from '@angular/router';
+import { Routes, RouterModule } from '@angular/router';
+import { AppPreloadingStrategy } from './app.preloading-strategy'
 import { StartComponent } from './components/start/start.component';
 import { TransactionComponent } from './components/transaction/transaction.component';
 import { BlockComponent } from './components/block/block.component';
@@ -12,7 +13,6 @@ import { TermsOfServiceComponent } from './components/terms-of-service/terms-of-
 import { PrivacyPolicyComponent } from './components/privacy-policy/privacy-policy.component';
 import { TrademarkPolicyComponent } from './components/trademark-policy/trademark-policy.component';
 import { BisqMasterPageComponent } from './components/bisq-master-page/bisq-master-page.component';
-import { SponsorComponent } from './components/sponsor/sponsor.component';
 import { PushTransactionComponent } from './components/push-transaction/push-transaction.component';
 import { BlocksList } from './components/blocks-list/blocks-list.component';
 import { LiquidMasterPageComponent } from './components/liquid-master-page/liquid-master-page.component';
@@ -22,14 +22,19 @@ import { AssetsComponent } from './components/assets/assets.component';
 import { AssetComponent } from './components/asset/asset.component';
 import { AssetsNavComponent } from './components/assets/assets-nav/assets-nav.component';
 
+const browserWindow = window || {};
+// @ts-ignore
+const browserWindowEnv = browserWindow.__env || {};
+
 let routes: Routes = [
-  { 
+  {
     path: 'testnet',
     children: [
       {
         path: '',
         pathMatch: 'full',
-        loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule)
+        loadChildren: () => import('./graphs/graphs.module').then(m => m.GraphsModule),
+        data: { preload: true },
       },
       {
         path: '',
@@ -67,7 +72,10 @@ let routes: Routes = [
           {
             path: 'address/:id',
             children: [],
-            component: AddressComponent
+            component: AddressComponent,
+            data: {
+              ogImage: true
+            }
           },
           {
             path: 'tx',
@@ -85,7 +93,10 @@ let routes: Routes = [
               children: [
               {
                 path: ':id',
-                component: BlockComponent
+                component: BlockComponent,
+                data: {
+                  ogImage: true
+                }
               },
             ],
           },
@@ -100,7 +111,8 @@ let routes: Routes = [
           },
           {
             path: 'docs',
-            loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule)
+            loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule),
+            data: { preload: true },
           },
           {
             path: 'api',
@@ -108,7 +120,8 @@ let routes: Routes = [
           },
           {
             path: 'lightning',
-            loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule)
+            loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule),
+            data: { preload: browserWindowEnv && browserWindowEnv.LIGHTNING === true },
           },
         ],
       },
@@ -170,7 +183,10 @@ let routes: Routes = [
           {
             path: 'address/:id',
             children: [],
-            component: AddressComponent
+            component: AddressComponent,
+            data: {
+              ogImage: true
+            }
           },
           {
             path: 'tx',
@@ -188,7 +204,10 @@ let routes: Routes = [
             children: [
               {
                 path: ':id',
-                component: BlockComponent
+                component: BlockComponent,
+                data: {
+                  ogImage: true
+                }
               },
             ],
           },
@@ -270,7 +289,10 @@ let routes: Routes = [
       {
         path: 'address/:id',
         children: [],
-        component: AddressComponent
+        component: AddressComponent,
+        data: {
+          ogImage: true
+        }
       },
       {
         path: 'tx',
@@ -288,7 +310,10 @@ let routes: Routes = [
         children: [
           {
             path: ':id',
-            component: BlockComponent
+            component: BlockComponent,
+            data: {
+              ogImage: true
+            }
           },
         ],
       },
@@ -316,12 +341,25 @@ let routes: Routes = [
     ],
   },
   {
-    path: 'status',
-    component: StatusViewComponent
+    path: 'preview',
+    children: [
+      {
+        path: '',
+        loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
+      },
+      {
+        path: 'testnet',
+        loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
+      },
+      {
+        path: 'signet',
+        loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
+      },
+    ],
   },
   {
-    path: 'sponsor',
-    component: SponsorComponent,
+    path: 'status',
+    component: StatusViewComponent
   },
   {
     path: '',
@@ -333,10 +371,6 @@ let routes: Routes = [
   },
 ];
 
-const browserWindow = window || {};
-// @ts-ignore
-const browserWindowEnv = browserWindow.__env || {};
-
 if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'bisq') {
   routes = [{
     path: '',
@@ -386,7 +420,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
             {
               path: 'address/:id',
               children: [],
-              component: AddressComponent
+              component: AddressComponent,
+              data: {
+                ogImage: true
+              }
             },
             {
               path: 'tx',
@@ -404,7 +441,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
               children: [
                 {
                   path: ':id',
-                  component: BlockComponent
+                  component: BlockComponent,
+                  data: {
+                    ogImage: true
+                  }
                 },
               ],
             },
@@ -490,7 +530,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
         {
           path: 'address/:id',
           children: [],
-          component: AddressComponent
+          component: AddressComponent,
+          data: {
+            ogImage: true
+          }
         },
         {
           path: 'tx',
@@ -508,7 +551,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
           children: [
             {
               path: ':id',
-              component: BlockComponent
+              component: BlockComponent,
+              data: {
+                ogImage: true
+              }
             },
           ],
         },
@@ -549,12 +595,21 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
       ],
     },
     {
-      path: 'status',
-      component: StatusViewComponent
+      path: 'preview',
+      children: [
+        {
+          path: '',
+          loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
+        },
+        {
+          path: 'testnet',
+          loadChildren: () => import('./previews.module').then(m => m.PreviewsModule)
+        },
+      ],
     },
     {
-      path: 'sponsor',
-      component: SponsorComponent,
+      path: 'status',
+      component: StatusViewComponent
     },
     {
       path: '',
@@ -572,8 +627,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
     initialNavigation: 'enabled',
     scrollPositionRestoration: 'enabled',
     anchorScrolling: 'enabled',
-    preloadingStrategy: PreloadAllModules
+    preloadingStrategy: AppPreloadingStrategy
   })],
 })
 export class AppRoutingModule { }
-

frontend/src/app/app.module.ts

@@ -1,14 +1,16 @@
 import { BrowserModule, BrowserTransferStateModule } from '@angular/platform-browser';
-import { NgModule } from '@angular/core';
+import { ModuleWithProviders, NgModule } from '@angular/core';
 import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
 import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
 import { AppRoutingModule } from './app-routing.module';
 import { AppComponent } from './components/app/app.component';
 import { ElectrsApiService } from './services/electrs-api.service';
 import { StateService } from './services/state.service';
+import { EnterpriseService } from './services/enterprise.service';
 import { WebsocketService } from './services/websocket.service';
 import { AudioService } from './services/audio.service';
 import { SeoService } from './services/seo.service';
+import { OpenGraphService } from './services/opengraph.service';
 import { SharedModule } from './shared/shared.module';
 import { StorageService } from './services/storage.service';
 import { HttpCacheInterceptor } from './services/http-cache.interceptor';
@@ -16,6 +18,24 @@ import { LanguageService } from './services/language.service';
 import { FiatShortenerPipe } from './shared/pipes/fiat-shortener.pipe';
 import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe';
 import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe';
+import { AppPreloadingStrategy } from './app.preloading-strategy';
+
+const providers = [
+  ElectrsApiService,
+  StateService,
+  WebsocketService,
+  AudioService,
+  SeoService,
+  OpenGraphService,
+  StorageService,
+  EnterpriseService,
+  LanguageService,
+  ShortenStringPipe,
+  FiatShortenerPipe,
+  CapAddressPipe,
+  AppPreloadingStrategy,
+  { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true }
+];
 
 @NgModule({
   declarations: [
@@ -29,19 +49,17 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe
     BrowserAnimationsModule,
     SharedModule,
   ],
-  providers: [
-    ElectrsApiService,
-    StateService,
-    WebsocketService,
-    AudioService,
-    SeoService,
-    StorageService,
-    LanguageService,
-    ShortenStringPipe,
-    FiatShortenerPipe,
-    CapAddressPipe,
-    { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true }
-  ],
+  providers: providers,
   bootstrap: [AppComponent]
 })
 export class AppModule { }
+
+@NgModule({})
+export class MempoolSharedModule{
+  static forRoot(): ModuleWithProviders<MempoolSharedModule> {
+    return {
+      ngModule: AppModule,
+      providers: providers
+    };
+  }
+}

frontend/src/app/app.preloading-strategy.ts

@@ -0,0 +1,10 @@
+import { PreloadingStrategy, Route } from '@angular/router';
+import { Observable, timer, mergeMap, of } from 'rxjs';
+
+export class AppPreloadingStrategy implements PreloadingStrategy {
+  preload(route: Route, load: Function): Observable<any> {
+      return route.data && route.data.preload 
+          ? timer(1500).pipe(mergeMap(() => load()))
+          : of(null);
+    }
+}

frontend/src/app/bisq/bisq-address/bisq-address.component.ts

@@ -1,11 +1,11 @@
 import { Component, OnInit, OnDestroy } from '@angular/core';
-import { SeoService } from 'src/app/services/seo.service';
+import { SeoService } from '../../services/seo.service';
 import { switchMap, filter, catchError } from 'rxjs/operators';
 import { ParamMap, ActivatedRoute } from '@angular/router';
 import { Subscription, of } from 'rxjs';
 import { BisqTransaction } from '../bisq.interfaces';
 import { BisqApiService } from '../bisq-api.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-address',

frontend/src/app/bisq/bisq-block/bisq-block.component.ts

@@ -1,14 +1,14 @@
 import { Component, OnInit, OnDestroy } from '@angular/core';
-import { BisqBlock } from 'src/app/bisq/bisq.interfaces';
+import { BisqBlock } from '../../bisq/bisq.interfaces';
 import { Location } from '@angular/common';
 import { BisqApiService } from '../bisq-api.service';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
 import { Subscription, of } from 'rxjs';
 import { switchMap, catchError } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { ElectrsApiService } from 'src/app/services/electrs-api.service';
+import { SeoService } from '../../services/seo.service';
+import { ElectrsApiService } from '../../services/electrs-api.service';
 import { HttpErrorResponse } from '@angular/common/http';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-block',

frontend/src/app/bisq/bisq-blocks/bisq-blocks.component.ts

@@ -3,9 +3,9 @@ import { BisqApiService } from '../bisq-api.service';
 import { switchMap, map, take, mergeMap, tap } from 'rxjs/operators';
 import { Observable } from 'rxjs';
 import { BisqBlock, BisqOutput, BisqTransaction } from '../bisq.interfaces';
-import { SeoService } from 'src/app/services/seo.service';
+import { SeoService } from '../../services/seo.service';
 import { ActivatedRoute, Router } from '@angular/router';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-blocks',

frontend/src/app/bisq/bisq-dashboard/bisq-dashboard.component.ts

@@ -1,9 +1,9 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { Observable, combineLatest, BehaviorSubject, of } from 'rxjs';
 import { map, share, switchMap } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
+import { WebsocketService } from '../../services/websocket.service';
 import { BisqApiService } from '../bisq-api.service';
 import { Trade } from '../bisq.interfaces';
 

frontend/src/app/bisq/bisq-main-dashboard/bisq-main-dashboard.component.ts

@@ -1,9 +1,9 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { Observable, combineLatest, BehaviorSubject, of } from 'rxjs';
 import { map, share, switchMap } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
+import { WebsocketService } from '../../services/websocket.service';
 import { BisqApiService } from '../bisq-api.service';
 import { Trade } from '../bisq.interfaces';
 

frontend/src/app/bisq/bisq-market/bisq-market.component.ts

@@ -3,8 +3,8 @@ import { FormBuilder, FormGroup } from '@angular/forms';
 import { ActivatedRoute, Router } from '@angular/router';
 import { combineLatest, merge, Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { SeoService } from 'src/app/services/seo.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { WebsocketService } from '../../services/websocket.service';
 import { BisqApiService } from '../bisq-api.service';
 import { OffersMarket, Trade } from '../bisq.interfaces';
 

frontend/src/app/bisq/bisq-stats/bisq-stats.component.ts

@@ -1,9 +1,9 @@
 import { Component, OnInit } from '@angular/core';
 import { BisqApiService } from '../bisq-api.service';
 import { BisqStats } from '../bisq.interfaces';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-stats',

frontend/src/app/bisq/bisq-transaction-details/bisq-transaction-details.component.ts

@@ -1,5 +1,5 @@
 import { Component, ChangeDetectionStrategy, Input, OnChanges } from '@angular/core';
-import { BisqTransaction } from 'src/app/bisq/bisq.interfaces';
+import { BisqTransaction } from '../../bisq/bisq.interfaces';
 
 @Component({
   selector: 'app-bisq-transaction-details',

frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts

@@ -1,15 +1,15 @@
 import { Component, OnInit, OnDestroy } from '@angular/core';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
-import { BisqTransaction } from 'src/app/bisq/bisq.interfaces';
+import { BisqTransaction } from '../../bisq/bisq.interfaces';
 import { switchMap, map, catchError } from 'rxjs/operators';
 import { of, Observable, Subscription } from 'rxjs';
-import { StateService } from 'src/app/services/state.service';
-import { Block, Transaction } from 'src/app/interfaces/electrs.interface';
+import { StateService } from '../../services/state.service';
+import { Block, Transaction } from '../../interfaces/electrs.interface';
 import { BisqApiService } from '../bisq-api.service';
-import { SeoService } from 'src/app/services/seo.service';
-import { ElectrsApiService } from 'src/app/services/electrs-api.service';
+import { SeoService } from '../../services/seo.service';
+import { ElectrsApiService } from '../../services/electrs-api.service';
 import { HttpErrorResponse } from '@angular/common/http';
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-transaction',

frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.ts

@@ -4,11 +4,11 @@ import { BisqTransaction, BisqOutput } from '../bisq.interfaces';
 import { Observable, Subscription } from 'rxjs';
 import { switchMap, map, tap } from 'rxjs/operators';
 import { BisqApiService } from '../bisq-api.service';
-import { SeoService } from 'src/app/services/seo.service';
+import { SeoService } from '../../services/seo.service';
 import { FormGroup, FormBuilder } from '@angular/forms';
 import { Router, ActivatedRoute } from '@angular/router';
-import { IMultiSelectOption, IMultiSelectSettings, IMultiSelectTexts } from 'src/app/components/ngx-bootstrap-multiselect/types'
-import { WebsocketService } from 'src/app/services/websocket.service';
+import { IMultiSelectOption, IMultiSelectSettings, IMultiSelectTexts } from '../../components/ngx-bootstrap-multiselect/types'
+import { WebsocketService } from '../../services/websocket.service';
 
 @Component({
   selector: 'app-bisq-transactions',

frontend/src/app/bisq/bisq-transfers/bisq-transfers.component.ts

@@ -1,9 +1,9 @@
 import { Component, OnInit, ChangeDetectionStrategy, Input, OnChanges } from '@angular/core';
-import { BisqTransaction } from 'src/app/bisq/bisq.interfaces';
-import { StateService } from 'src/app/services/state.service';
+import { BisqTransaction } from '../../bisq/bisq.interfaces';
+import { StateService } from '../../services/state.service';
 import { map } from 'rxjs/operators';
 import { Observable } from 'rxjs';
-import { Block } from 'src/app/interfaces/electrs.interface';
+import { Block } from '../../interfaces/electrs.interface';
 
 @Component({
   selector: 'app-bisq-transfers',

frontend/src/app/bisq/bsq-amount/bsq-amount.component.ts

@@ -1,5 +1,5 @@
 import { Component, OnInit, ChangeDetectionStrategy, Input } from '@angular/core';
-import { StateService } from 'src/app/services/state.service';
+import { StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
 
 @Component({

frontend/src/app/bitcoin.utils.ts

@@ -5,64 +5,157 @@ const P2SH_P2WSH_COST  = 35 * 4; // the WU cost for the non-witness part of P2SH
 
 export function calcSegwitFeeGains(tx: Transaction) {
   // calculated in weight units
-  let realizedGains = 0;
-  let potentialBech32Gains = 0;
-  let potentialP2shGains = 0;
+  let realizedSegwitGains = 0;
+  let potentialSegwitGains = 0;
+  let potentialP2shSegwitGains = 0;
+  let potentialTaprootGains = 0;
+  let realizedTaprootGains = 0;
 
   for (const vin of tx.vin) {
     if (!vin.prevout) { continue; }
 
-    const isP2pkh = vin.prevout.scriptpubkey_type === 'p2pkh';
-    const isP2sh  = vin.prevout.scriptpubkey_type === 'p2sh';
-    const isP2wsh = vin.prevout.scriptpubkey_type === 'v0_p2wsh';
-    const isP2wpkh = vin.prevout.scriptpubkey_type === 'v0_p2wpkh';
-    const isP2tr  = vin.prevout.scriptpubkey_type === 'v1_p2tr';
+    const isP2pk         = vin.prevout.scriptpubkey_type === 'p2pk';
+    // const isBareMultisig = vin.prevout.scriptpubkey_type === 'multisig'; // type will be unknown, so use the multisig helper from the address labels
+    const isBareMultisig = !!parseMultisigScript(vin.prevout.scriptpubkey_asm);
+    const isP2pkh        = vin.prevout.scriptpubkey_type === 'p2pkh';
+    const isP2sh         = vin.prevout.scriptpubkey_type === 'p2sh';
+    const isP2wsh        = vin.prevout.scriptpubkey_type === 'v0_p2wsh';
+    const isP2wpkh       = vin.prevout.scriptpubkey_type === 'v0_p2wpkh';
+    const isP2tr         = vin.prevout.scriptpubkey_type === 'v1_p2tr';
 
     const op = vin.scriptsig ? vin.scriptsig_asm.split(' ')[0] : null;
-    const isP2sh2Wpkh = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_22';
-    const isP2sh2Wsh = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_34';
+    const isP2shP2Wpkh = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_22';
+    const isP2shP2Wsh  = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_34';
 
     switch (true) {
-      // Native Segwit - P2WPKH/P2WSH (Bech32)
+      // Native Segwit - P2WPKH/P2WSH/P2TR
       case isP2wpkh:
       case isP2wsh:
       case isP2tr:
         // maximal gains: the scriptSig is moved entirely to the witness part
-        realizedGains += witnessSize(vin) * 3;
+        // if taproot is used savings are 42 WU higher because it produces smaller signatures and doesn't require a pubkey in the witness
+        // this number is explained above `realizedTaprootGains += 42;`
+        realizedSegwitGains += (witnessSize(vin) + (isP2tr ? 42 : 0)) * 3;
         // XXX P2WSH output creation is more expensive, should we take this into consideration?
         break;
 
       // Backward compatible Segwit - P2SH-P2WPKH
-      case isP2sh2Wpkh:
-        // the scriptSig is moved to the witness, but we have extra 21 extra non-witness bytes (48 WU)
-        realizedGains += witnessSize(vin) * 3 - P2SH_P2WPKH_COST;
-        potentialBech32Gains += P2SH_P2WPKH_COST;
+      case isP2shP2Wpkh:
+        // the scriptSig is moved to the witness, but we have extra 21 extra non-witness bytes (84 WU)
+        realizedSegwitGains += witnessSize(vin) * 3 - P2SH_P2WPKH_COST;
+        potentialSegwitGains += P2SH_P2WPKH_COST;
         break;
 
       // Backward compatible Segwit - P2SH-P2WSH
-      case isP2sh2Wsh:
-        // the scriptSig is moved to the witness, but we have extra 35 extra non-witness bytes
-        realizedGains += witnessSize(vin) * 3 - P2SH_P2WSH_COST;
-        potentialBech32Gains += P2SH_P2WSH_COST;
+      case isP2shP2Wsh:
+        // the scriptSig is moved to the witness, but we have extra 35 extra non-witness bytes (140 WU)
+        realizedSegwitGains += witnessSize(vin) * 3 - P2SH_P2WSH_COST;
+        potentialSegwitGains += P2SH_P2WSH_COST;
         break;
 
-      // Non-segwit P2PKH/P2SH
+      // Non-segwit P2PKH/P2SH/P2PK/bare multisig
       case isP2pkh:
       case isP2sh:
-        const fullGains = scriptSigSize(vin) * 3;
-        potentialBech32Gains += fullGains;
-        potentialP2shGains += fullGains - (isP2pkh ? P2SH_P2WPKH_COST : P2SH_P2WSH_COST);
+      case isP2pk:
+      case isBareMultisig: {
+        let fullGains = scriptSigSize(vin) * 3;
+        if (isBareMultisig) {
+          // a _bare_ multisig has the keys in the output script, but P2SH and P2WSH require them to be in the scriptSig/scriptWitness
+          fullGains -= vin.prevout.scriptpubkey.length / 2;
+        }
+        potentialSegwitGains += fullGains;
+        potentialP2shSegwitGains += fullGains - (isP2pkh ? P2SH_P2WPKH_COST : P2SH_P2WSH_COST);
         break;
+      }
+    }
 
-    // TODO: should we also consider P2PK and pay-to-bare-script (non-p2sh-wrapped) as upgradable to P2WPKH and P2WSH?
+    if (isP2tr) {
+      if (vin.witness.length === 1) {
+        // key path spend
+        // we don't know if this was a multisig or single sig (the goal of taproot :)),
+        // so calculate fee savings by comparing to the cheapest single sig input type: P2WPKH and say "saved at least ...%"
+        // the witness size of P2WPKH is 1 (stack size) + 1 (size) + 72 (low s signature) + 1 (size) + 33 (pubkey) = 108 WU
+        // the witness size of key path P2TR is 1 (stack size) + 1 (size) + 64 (signature) = 66 WU
+        realizedTaprootGains += 42;
+      } else {
+        // script path spend
+        // complex scripts with multiple spending paths can often be made around 2x to 3x smaller with the Taproot script tree
+        // because only the hash of the alternative spending path has the be in the witness data, not the entire script,
+        // but only assumptions can be made because the scripts themselves are unknown (again, the goal of taproot :))
+        // TODO maybe add some complex scripts that are specified somewhere, so that size is known, such as lightning scripts
+      }
+    } else {
+      const script = isP2shP2Wsh || isP2wsh ? vin.inner_witnessscript_asm : vin.inner_redeemscript_asm;
+      let replacementSize: number;
+      if (
+        // single sig
+        isP2pk || isP2pkh || isP2wpkh || isP2shP2Wpkh ||
+        // multisig
+        isBareMultisig || parseMultisigScript(script)
+      ) {
+        // the scriptSig and scriptWitness can all be replaced by a 66 witness WU with taproot
+        replacementSize = 66;
+      } else if (script) {
+        // not single sig, not multisig: the complex scripts
+        // rough calculations on spending paths
+        // every OP_IF and OP_NOTIF indicates an _extra_ spending path, so add 1
+        const spendingPaths = script.split(' ').filter(op => /^(OP_IF|OP_NOTIF)$/g.test(op)).length + 1;
+        // now assume the script could have been split in ${spendingPaths} equal tapleaves
+        replacementSize = script.length / 2 / spendingPaths +
+        // but account for the leaf and branch hashes and internal key in the control block
+          32 * Math.log2((spendingPaths - 1) || 1) + 33;
+      }
+      potentialTaprootGains += witnessSize(vin) + scriptSigSize(vin) * 4 - replacementSize;
     }
   }
 
   // returned as percentage of the total tx weight
-  return { realizedGains: realizedGains / (tx.weight + realizedGains) // percent of the pre-segwit tx size
-         , potentialBech32Gains: potentialBech32Gains / tx.weight
-         , potentialP2shGains: potentialP2shGains / tx.weight
-         };
+  return {
+    realizedSegwitGains: realizedSegwitGains / (tx.weight + realizedSegwitGains), // percent of the pre-segwit tx size
+    potentialSegwitGains: potentialSegwitGains / tx.weight,
+    potentialP2shSegwitGains: potentialP2shSegwitGains / tx.weight,
+    potentialTaprootGains: potentialTaprootGains / tx.weight,
+    realizedTaprootGains: realizedTaprootGains / (tx.weight + realizedTaprootGains)
+  };
+}
+
+/** extracts m and n from a multisig script (asm), returns nothing if it is not a multisig script */
+export function parseMultisigScript(script: string): void | { m: number, n: number } {
+  if (!script) {
+    return;
+  }
+  const ops = script.split(' ');
+  if (ops.length < 3 || ops.pop() !== 'OP_CHECKMULTISIG') {
+    return;
+  }
+  const opN = ops.pop();
+  if (!opN.startsWith('OP_PUSHNUM_')) {
+    return;
+  }
+  const n = parseInt(opN.match(/[0-9]+/)[0], 10);
+  if (ops.length < n * 2 + 1) {
+    return;
+  }
+  // pop n public keys
+  for (let i = 0; i < n; i++) {
+    if (!/^0((2|3)\w{64}|4\w{128})$/.test(ops.pop())) {
+      return;
+    }
+    if (!/^OP_PUSHBYTES_(33|65)$/.test(ops.pop())) {
+      return;
+    }
+  }
+  const opM = ops.pop();
+  if (!opM.startsWith('OP_PUSHNUM_')) {
+    return;
+  }
+  const m = parseInt(opM.match(/[0-9]+/)[0], 10);
+
+  if (ops.length) {
+    return;
+  }
+
+  return { m, n };
 }
 
 // https://github.com/shesek/move-decimal-point
@@ -101,12 +194,12 @@ export function moveDec(num: number, n: number) {
   return neg + (int || '0') + (frac.length ? '.' + frac : '');
 }
 
-function zeros(n) {
+function zeros(n: number) {
   return new Array(n + 1).join('0');
 }
 
 // Formats a number for display. Treats the number as a string to avoid rounding errors.
-export const formatNumber = (s, precision = null) => {
+export const formatNumber = (s: number | string, precision: number | null = null) => {
   let [ whole, dec ] = s.toString().split('.');
 
   // divide numbers into groups of three separated with a thin space (U+202F, "NARROW NO-BREAK SPACE"),
@@ -128,31 +221,31 @@ export const formatNumber = (s, precision = null) => {
 };
 
 // Utilities for segwitFeeGains
-const witnessSize = (vin: Vin) => vin.witness.reduce((S, w) => S + (w.length / 2), 0);
+const witnessSize = (vin: Vin) => vin.witness ? vin.witness.reduce((S, w) => S + (w.length / 2), 0) : 0;
 const scriptSigSize = (vin: Vin) => vin.scriptsig ? vin.scriptsig.length / 2 : 0;
 
 // Power of ten wrapper
-export function selectPowerOfTen(val: number) {
+export function selectPowerOfTen(val: number): { divider: number, unit: string } {
   const powerOfTen = {
     exa: Math.pow(10, 18),
     peta: Math.pow(10, 15),
-    terra: Math.pow(10, 12),
+    tera: Math.pow(10, 12),
     giga: Math.pow(10, 9),
     mega: Math.pow(10, 6),
     kilo: Math.pow(10, 3),
   };
 
-  let selectedPowerOfTen;
+  let selectedPowerOfTen: { divider: number, unit: string };
   if (val < powerOfTen.kilo) {
     selectedPowerOfTen = { divider: 1, unit: '' }; // no scaling
   } else if (val < powerOfTen.mega) {
     selectedPowerOfTen = { divider: powerOfTen.kilo, unit: 'k' };
   } else if (val < powerOfTen.giga) {
     selectedPowerOfTen = { divider: powerOfTen.mega, unit: 'M' };
-  } else if (val < powerOfTen.terra) {
+  } else if (val < powerOfTen.tera) {
     selectedPowerOfTen = { divider: powerOfTen.giga, unit: 'G' };
   } else if (val < powerOfTen.peta) {
-    selectedPowerOfTen = { divider: powerOfTen.terra, unit: 'T' };
+    selectedPowerOfTen = { divider: powerOfTen.tera, unit: 'T' };
   } else if (val < powerOfTen.exa) {
     selectedPowerOfTen = { divider: powerOfTen.peta, unit: 'P' };
   } else {
@@ -160,4 +253,4 @@ export function selectPowerOfTen(val: number) {
   }
 
   return selectedPowerOfTen;
-}
+}

frontend/src/app/components/about/about.component.html

@@ -25,12 +25,6 @@
     </a>
   </div>
 
-  <br><br>
-  <div class="sponsor-button">
-    <button [hidden]="showNavigateToSponsor" type="button" class="btn btn-primary" (click)="sponsor()" i18n="about.become-a-sponsor">Become a sponsor ❤️</button>
-    <ng-container *ngIf="showNavigateToSponsor" i18n="about.navigate-to-sponsor">Navigate to <a href="https://mempool.space/sponsor" target="_blank">https://mempool.space/sponsor</a> to sponsor</ng-container>
-  </div>
-
   <div class="enterprise-sponsor">
     <h3 i18n="about.sponsors.enterprise.withRocket">Enterprise Sponsors 🚀</h3>
     <div class="wrapper">
@@ -193,8 +187,8 @@
     </div>
   </div>
 
-  <div class="selfhosted-integrations-sponsor">
-    <h3 i18n="about.self-hosted-integrations">Self-Hosted Integrations</h3>
+  <div class="community-integrations-sponsor">
+    <h3 i18n="about.community-integrations">Community Integrations</h3>
     <div class="wrapper">
       <a href="https://github.com/getumbrel/umbrel" target="_blank" title="Umbrel">
         <img class="image" src="/resources/profile/umbrel.png" />
@@ -224,18 +218,24 @@
         <img class="image" src="/resources/profile/start9.png" />
         <span>EmbassyOS</span>
       </a>
-    </div>
-  </div>
-
-  <div class="community-integrations-sponsor">
-    <h3 i18n="about.wallet-integrations">Wallet Integrations</h3>
-    <div class="wrapper">
+      <a href="https://github.com/btcpayserver/btcpayserver" target="_blank" title="BTCPay Server">
+        <img class="image" src="/resources/profile/btcpayserver.svg" />
+        <span>BTCPay</span>
+      </a>
       <a href="https://github.com/bisq-network/bisq" target="_blank" title="Bisq">
         <img class="image" src="/resources/profile/bisq_network.png" />
         <span>Bisq</span>
       </a>
+      <a href="https://github.com/BlueWallet/BlueWallet" target="_blank" title="BlueWallet">
+        <img class="image" src="/resources/profile/bluewallet.png" />
+        <span>BlueWallet</span>
+      </a>
+      <a href="https://github.com/muun/apollo" target="_blank" title="Muun Wallet">
+        <img class="image" src="/resources/profile/muun.png" />
+        <span>Muun</span>
+      </a>
       <a href="https://github.com/spesmilo/electrum" target="_blank" title="Electrum Wallet">
-        <img class="image" src="/resources/profile/electrum.jpg" />
+        <img class="image" src="/resources/profile/electrum.png" />
         <span>Electrum</span>
       </a>
       <a href="https://github.com/cryptoadvance/specter-desktop" target="_blank" title="Specter Wallet">
@@ -250,18 +250,14 @@
         <img class="image" src="/resources/profile/phoenix.jpg" />
         <span>Phoenix</span>
       </a>
+      <a href="https://github.com/lnbits/lnbits-legend" target="_blank" title="LNbits">
+        <img class="image" src="/resources/profile/lnbits.svg" />
+        <span>LNBits</span>
+      </a>
       <a href="https://github.com/layer2tech/mercury-wallet" target="_blank" title="Mercury Wallet">
         <img class="image" src="/resources/profile/mercury.svg" />
         <span>Mercury</span>
       </a>
-      <a href="https://github.com/muun/apollo" target="_blank" title="Muun Wallet">
-        <img class="image" src="/resources/profile/muun.png" />
-        <span>Muun</span>
-      </a>
-      <a href="https://github.com/BlueWallet/BlueWallet" target="_blank" title="BlueWallet">
-        <img class="image" src="/resources/profile/bluewallet.png" />
-        <span>BlueWallet</span>
-      </a>
       <a href="https://github.com/hsjoberg/blixt-wallet" target="_blank" title="Blixt Wallet">
         <img class="image" src="/resources/profile/blixt.png" />
         <span>Blixt</span>
@@ -297,7 +293,7 @@
   </div>
 
   <ng-container *ngIf="translators$ | async | keyvalue as translators else loadingSponsors">
-    <div class="community-sponsor">
+    <div class="project-translators">
       <h3 i18n="about.translators">Project Translators</h3>
       <div class="wrapper">
         <ng-template ngFor let-translator [ngForOf]="translators">

frontend/src/app/components/about/about.component.scss

@@ -43,7 +43,6 @@
   .alliances,
   .enterprise-sponsor,
   .community-integrations-sponsor,
-  .selfhosted-integrations-sponsor,
   .maintainers {
     margin-top: 68px;
     margin-bottom: 68px;
@@ -53,7 +52,7 @@
     margin-bottom: 50px;
   }
 
-  .community-sponsor {
+  .community-sponsor, .project-translators {
     display: flex;
     flex-direction: column;
     .wrapper {
@@ -67,6 +66,13 @@
     }
   }
 
+  .community-sponsor {
+    img {
+      width: 67px;
+      height: 67px;
+    }
+  }
+
   .alliances {
     margin-bottom: 100px;
     a {
@@ -108,8 +114,8 @@
   .enterprise-sponsor,
   .contributors,
   .community-sponsor,
+  .project-translators,
   .community-integrations-sponsor,
-  .selfhosted-integrations-sponsor,
   .maintainers {
     .wrapper {
       display: inline-block;
@@ -132,7 +138,7 @@
     }
   }
 
-  .community-sponsor .wrapper {
+  .community-sponsor .wrapper, .project-translators .wrapper {
     margin: 10px auto 20px;
     a img {
       margin: 6px;
@@ -185,6 +191,6 @@
 }
 
 .community-integrations-sponsor {
-  max-width: 830px;
+  max-width: 970px;
   margin: auto;
 }

frontend/src/app/components/about/about.component.ts

@@ -1,13 +1,13 @@
 import { ChangeDetectionStrategy, Component, Inject, LOCALE_ID, OnInit } from '@angular/core';
 import { WebsocketService } from '../../services/websocket.service';
-import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
+import { SeoService } from '../../services/seo.service';
+import { StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
-import { ApiService } from 'src/app/services/api.service';
-import { IBackendInfo } from 'src/app/interfaces/websocket.interface';
+import { ApiService } from '../../services/api.service';
+import { IBackendInfo } from '../../interfaces/websocket.interface';
 import { Router } from '@angular/router';
 import { map } from 'rxjs/operators';
-import { ITranslators } from 'src/app/interfaces/node-api.interface';
+import { ITranslators } from '../../interfaces/node-api.interface';
 
 @Component({
   selector: 'app-about',
@@ -61,9 +61,9 @@ export class AboutComponent implements OnInit {
     );
   }
 
-  sponsor() {
+  sponsor(): void {
     if (this.officialMempoolSpace && this.stateService.env.BASE_MODULE === 'mempool') {
-      this.router.navigateByUrl('/sponsor');
+      this.router.navigateByUrl('/enterprise');
     } else {
       this.showNavigateToSponsor = true;
     }

frontend/src/app/components/address-labels/address-labels.component.html

@@ -1,9 +1,16 @@
-<a *ngIf="channel; else default" [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">
-  <span
-    *ngIf="label"
-    class="badge badge-pill badge-warning"
-  >{{ label }}</span>
-</a>
+<ng-template [ngIf]="channel" [ngIfElse]="default">
+  <div>
+    <div class="badge-positioner">
+      <a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">
+        <span 
+          *ngIf="label"
+          class="badge badge-pill badge-warning"
+        >{{ label }}</span>
+      </a>
+    </div>
+    &nbsp;
+  </div>
+</ng-template>
 
 <ng-template #default>
   <span

frontend/src/app/components/address-labels/address-labels.component.scss

@@ -1,3 +1,7 @@
 .badge {
   margin-right: 2px;
 }
+
+.badge-positioner {
+  position: absolute;
+}

frontend/src/app/components/address-labels/address-labels.component.ts

@@ -1,6 +1,7 @@
 import { Component, ChangeDetectionStrategy, Input, OnChanges } from '@angular/core';
 import { Vin, Vout } from '../../interfaces/electrs.interface';
-import { StateService } from 'src/app/services/state.service';
+import { StateService } from '../../services/state.service';
+import { parseMultisigScript } from '../../bitcoin.utils';
 
 @Component({
   selector: 'app-address-labels',
@@ -98,41 +99,11 @@ export class AddressLabelsComponent implements OnChanges {
   }
 
   detectMultisig(script: string) {
-    if (!script) {
-      return;
-    }
-    const ops = script.split(' ');
-    if (ops.length < 3 || ops.pop() !== 'OP_CHECKMULTISIG') {
-      return;
-    }
-    const opN = ops.pop();
-    if (!opN.startsWith('OP_PUSHNUM_')) {
-      return;
-    }
-    const n = parseInt(opN.match(/[0-9]+/)[0], 10);
-    if (ops.length < n * 2 + 1) {
-      return;
-    }
-    // pop n public keys
-    for (let i = 0; i < n; i++) {
-      if (!/^0((2|3)\w{64}|4\w{128})$/.test(ops.pop())) {
-        return;
-      }
-      if (!/^OP_PUSHBYTES_(33|65)$/.test(ops.pop())) {
-        return;
-      }
-    }
-    const opM = ops.pop();
-    if (!opM.startsWith('OP_PUSHNUM_')) {
-      return;
-    }
-    const m = parseInt(opM.match(/[0-9]+/)[0], 10);
+    const ms = parseMultisigScript(script);
 
-    if (ops.length) {
-      return;
+    if (ms) {
+      this.label = $localize`:@@address-label.multisig:Multisig ${ms.m}:multisigM: of ${ms.n}:multisigN:`;
     }
-
-    this.label = $localize`:@@address-label.multisig:Multisig ${m}:multisigM: of ${n}:multisigN:`;
   }
 
   handleVout() {