From fc73dcc3444b73b31110c45734a0155524302685 Mon Sep 17 00:00:00 2001 From: Stephan Oeste Date: Mon, 7 Aug 2023 20:08:30 +0200 Subject: [PATCH 001/152] Fixing proxy_buffer_size error nginx.conf --- production/nginx/http-proxy-cache.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/production/nginx/http-proxy-cache.conf b/production/nginx/http-proxy-cache.conf index f870939b3..60c6d4f82 100644 --- a/production/nginx/http-proxy-cache.conf +++ b/production/nginx/http-proxy-cache.conf @@ -3,3 +3,4 @@ proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=100m; proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=100m; types_hash_max_size 2048; +proxy_buffer_size 8k; From c2cad416e23c053d381e2fd84f9f980862f9ab11 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Wed, 9 Aug 2023 22:33:59 -0700 Subject: [PATCH 002/152] Add frontend dependency groups to dependabot --- .github/dependabot.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a824900f4..8352de555 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,20 @@ updates: - package-ecosystem: npm directory: "/frontend" versioning-strategy: increase + groups: + frontend-angular-dependencies: + patterns: + - "@angular*" + - "@ng-*" + - "ngx-*" + frontend-jest-dependencies: + patterns: + - "@types/jest" + - "jest" + frontend-eslint-dependencies: + patterns: + - "@typescript-eslint*" + - "eslint" schedule: interval: daily open-pull-requests-limit: 10 From 45ba50e82cb6f884583bb75c1aed44c5d1036269 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 10 Aug 2023 16:26:07 +0900 Subject: [PATCH 003/152] Change citadel link on about page --- frontend/src/app/components/about/about.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index e9d5ec3b2..b9ef7a8cf 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -257,7 +257,7 @@ RoninDojo - + Citadel From 8c21d106fc58145a13d238516cf77844513f8391 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 14 Aug 2023 02:26:34 +0900 Subject: [PATCH 004/152] Fix stacked mempool blocks layout & width --- .../mempool-blocks/mempool-blocks.component.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index 2269d38a9..b25be2dd6 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -31,6 +31,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { @Input() spotlight: number = 0; @Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`; @Input() allBlocks: boolean = false; + @Input() forceRtl: boolean = false; mempoolWidth: number = 0; @Output() widthChange: EventEmitter = new EventEmitter(); @@ -102,7 +103,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { } this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => { - this.timeLtr = !!ltr; + this.timeLtr = !this.forceRtl && !!ltr; this.cd.markForCheck(); }); @@ -283,7 +284,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { } reduceEmptyBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] { - const innerWidth = this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2; + const innerWidth = this.containerWidth || (this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2); let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT; if (!this.allBlocks) { blocksAmount = Math.min(this.stateService.env.MEMPOOL_BLOCKS_AMOUNT, Math.floor(innerWidth / (this.blockWidth + this.blockPadding))); @@ -306,7 +307,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { } reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] { - const innerWidth = this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2; + const innerWidth = this.containerWidth || (this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2); let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT; if (this.count) { blocksAmount = 8; @@ -316,7 +317,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { while (blocks.length > blocksAmount) { const block = blocks.pop(); if (!this.count) { - const lastBlock = blocks[0]; + const lastBlock = blocks[blocks.length - 1]; lastBlock.blockSize += block.blockSize; lastBlock.blockVSize += block.blockVSize; lastBlock.nTx += block.nTx; @@ -327,7 +328,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { } } if (blocks.length) { - blocks[0].isStack = blocks[0].blockVSize > this.stateService.blockVSize; + blocks[blocks.length - 1].isStack = blocks[blocks.length - 1].blockVSize > this.stateService.blockVSize; } return blocks; } From 13f6f9f9e5a5db7eae0197b1bf3f079324c01cf0 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 14 Aug 2023 17:52:15 +0900 Subject: [PATCH 005/152] Select unfurl mode with X-Unfurl-Type header --- unfurler/src/index.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index f3acd61bf..ae751d1f5 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -250,25 +250,25 @@ class Server { || rawPath.startsWith('/api/v1/translators/images') || rawPath.startsWith('/resources/profile') ) { - if (isSearchCrawler(req.headers['user-agent'])) { + if (isPreviewCrawler(req)) { + res.status(404).send(); + return; + } else { if (this.secureHost) { https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } else { http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } return; - } else { - res.status(404).send(); - return; } } let result = ''; try { - if (isSearchCrawler(req.headers['user-agent'])) { - result = await this.renderSEOPage(rawPath); - } else { + if (isPreviewCrawler(req)) { result = await this.renderUnfurlMeta(rawPath); + } else { + result = await this.renderSEOPage(rawPath); } if (result && result.length) { if (result === '404') { @@ -349,6 +349,6 @@ function capitalize(str) { } } -function isSearchCrawler(useragent: string): boolean { - return /googlebot|applebot|bingbot/i.test(useragent); +function isPreviewCrawler(req: Request): boolean { + return req?.header('X-Unfurl-Type') === 'preview'; } From 5511795fbb4001ec44e265ab0a7cfab1de2a6e95 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 14 Aug 2023 19:26:08 +0900 Subject: [PATCH 006/152] Fix mempool slide on page change --- .../mempool-blocks/mempool-blocks.component.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index b25be2dd6..484389cd3 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -115,11 +115,6 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { }); this.reduceEmptyBlocksToFitScreen(this.mempoolEmptyBlocks); - this.mempoolBlocks.map(() => { - this.updateMempoolBlockStyles(); - this.calculateTransactionPosition(); - }); - this.reduceMempoolBlocksToFitScreen(this.mempoolBlocks); this.isTabHiddenSubscription = this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden); this.loadingBlocks$ = combineLatest([ this.stateService.isLoadingWebSocket$, @@ -207,14 +202,17 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { if (!block) { return; } + + const isNewBlock = block.height > this.chainTip; + if (this.chainTip === -1) { this.animateEntry = block.height === this.stateService.latestBlockHeight; } else { - this.animateEntry = block.height > this.chainTip; + this.animateEntry = isNewBlock; } this.chainTip = this.stateService.latestBlockHeight; - if ((block?.extras?.similarity == null || block?.extras?.similarity > 0.5) && !this.tabHidden) { + if (isNewBlock && (block?.extras?.similarity == null || block?.extras?.similarity > 0.5) && !this.tabHidden) { this.blockIndex++; } }); From 83a92a7e3a709c9e192e195f25b0e6c370cc5b55 Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 16 Aug 2023 03:27:09 +0900 Subject: [PATCH 007/152] ops: Add nginx config for /unfurler and /slurper prefixes --- production/nginx/http-proxy-cache.conf | 8 ++-- production/nginx/nginx.conf | 4 +- production/nginx/server-common.conf | 55 +++++++++++++++++++++----- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/production/nginx/http-proxy-cache.conf b/production/nginx/http-proxy-cache.conf index f870939b3..4d0e6614e 100644 --- a/production/nginx/http-proxy-cache.conf +++ b/production/nginx/http-proxy-cache.conf @@ -1,5 +1,7 @@ # proxy cache -proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=100m; -proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=100m; -proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=100m; +proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=600s max_size=200m; types_hash_max_size 2048; diff --git a/production/nginx/nginx.conf b/production/nginx/nginx.conf index 5861067e8..c08005d14 100644 --- a/production/nginx/nginx.conf +++ b/production/nginx/nginx.conf @@ -23,8 +23,8 @@ http { include mempool/production/nginx/http-language.conf; # match preview/unfurl bot user-agents - map $http_user_agent $unfurlbot { - default 0; + map $http_user_agent $unfurlprefix { + default ""; } # mempool configuration diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index 68f3b6971..3406e3450 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -48,8 +48,8 @@ add_header Vary Cookie; # for exact / requests, redirect based on $lang # cache redirect for 5 minutes location = / { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } if ($lang != '') { return 302 $scheme://$host/$lang/; @@ -65,15 +65,15 @@ location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ { } # cache everything else for 5 minutes location ~ ^/([a-z][a-z])$ { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } try_files $uri /$1/index.html /en-US/index.html =404; expires 5m; } location ~ ^/([a-z][a-z])/ { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } try_files $uri /$1/index.html /en-US/index.html =404; expires 5m; @@ -104,16 +104,53 @@ location /preview { } # unfurl renderer location ^~ /render { - proxy_pass $mempoolSpaceUnfurler; + try_files /dev/null @mempool-space-unfurler; expires 10m; } +# unfurl handler +location /unfurl/ { + try_files /dev/null @mempool-space-unfurler; +} +location /slurp/ { + try_files /dev/null @mempool-space-slurper; +} # catch-all for all URLs i.e. /address/foo /tx/foo /block/000 # cache 5 minutes since they change frequently location / { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; expires 5m; } + +location @mempool-space-unfurler { + proxy_pass $mempoolSpaceUnfurler; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache unfurler; + proxy_cache_valid 200 10m; + proxy_redirect off; +} + +location @mempool-space-slurper { + proxy_pass $mempoolSpaceUnfurler; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache slurper; + proxy_cache_valid 200 10m; + proxy_redirect off; +} From 0bfd01e732cf104b3a457016648219cff2ec748b Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 16 Aug 2023 03:51:10 +0900 Subject: [PATCH 008/152] ops: Add unfurl/slurp queries to nginx warm cache script --- production/nginx-cache-warmer | 285 ++++++++++++++++++++-------------- 1 file changed, 167 insertions(+), 118 deletions(-) diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index 6a82871a5..fe3587c93 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -1,140 +1,189 @@ #!/usr/bin/env zsh +delay=0.15 hostname=$(hostname) slugs=(`curl -sSL https://${hostname}/api/v1/mining/pools/3y|jq -r -S '(.pools[].slug)'`) +warmSlurp() +{ + echo "$1" + curl -i -s -H 'User-Agent: Googlebot' "$1" | head -1 +} + +warmUnfurl() +{ + echo "$1" + curl -i -s -H 'User-Agent: Twitterbot' "$1" | head -1 +} + warm() { echo "$1" curl -i -s "$1" | head -1 } -while true -do for url in / \ - '/api/v1/blocks' \ - '/api/v1/statistics/2h' \ - '/api/v1/statistics/24h' \ - '/api/v1/statistics/1w' \ - '/api/v1/statistics/1m' \ - '/api/v1/statistics/3m' \ - '/api/v1/statistics/6m' \ - '/api/v1/statistics/1y' \ - '/api/v1/statistics/2y' \ - '/api/v1/statistics/3y' \ - '/api/v1/statistics/4y' \ - '/api/v1/statistics/all' \ - '/api/v1/mining/pools/24h' \ - '/api/v1/mining/pools/3d' \ - '/api/v1/mining/pools/1w' \ - '/api/v1/mining/pools/1m' \ - '/api/v1/mining/pools/3m' \ - '/api/v1/mining/pools/6m' \ - '/api/v1/mining/pools/1y' \ - '/api/v1/mining/pools/2y' \ - '/api/v1/mining/pools/3y' \ - '/api/v1/mining/pools/all' \ - '/api/v1/mining/hashrate/3m' \ - '/api/v1/mining/hashrate/6m' \ - '/api/v1/mining/hashrate/1y' \ - '/api/v1/mining/hashrate/2y' \ - '/api/v1/mining/hashrate/3y' \ - '/api/v1/mining/hashrate/all' \ - '/api/v1/mining/hashrate/pools/3m' \ - '/api/v1/mining/hashrate/pools/6m' \ - '/api/v1/mining/hashrate/pools/1y' \ - '/api/v1/mining/hashrate/pools/2y' \ - '/api/v1/mining/hashrate/pools/3y' \ - '/api/v1/mining/hashrate/pools/all' \ - '/api/v1/mining/reward-stats/144' \ - '/api/v1/mining/blocks/fees/24h' \ - '/api/v1/mining/blocks/fees/3d' \ - '/api/v1/mining/blocks/fees/1w' \ - '/api/v1/mining/blocks/fees/1m' \ - '/api/v1/mining/blocks/fees/3m' \ - '/api/v1/mining/blocks/fees/6m' \ - '/api/v1/mining/blocks/fees/1y' \ - '/api/v1/mining/blocks/fees/2y' \ - '/api/v1/mining/blocks/fees/3y' \ - '/api/v1/mining/blocks/fees/all' \ - '/api/v1/mining/blocks/rewards/24h' \ - '/api/v1/mining/blocks/rewards/3d' \ - '/api/v1/mining/blocks/rewards/1w' \ - '/api/v1/mining/blocks/rewards/1m' \ - '/api/v1/mining/blocks/rewards/3m' \ - '/api/v1/mining/blocks/rewards/6m' \ - '/api/v1/mining/blocks/rewards/1y' \ - '/api/v1/mining/blocks/rewards/2y' \ - '/api/v1/mining/blocks/rewards/3y' \ - '/api/v1/mining/blocks/rewards/all' \ - '/api/v1/mining/blocks/fee-rates/24h' \ - '/api/v1/mining/blocks/fee-rates/3d' \ - '/api/v1/mining/blocks/fee-rates/1w' \ - '/api/v1/mining/blocks/fee-rates/1m' \ - '/api/v1/mining/blocks/fee-rates/3m' \ - '/api/v1/mining/blocks/fee-rates/6m' \ - '/api/v1/mining/blocks/fee-rates/1y' \ - '/api/v1/mining/blocks/fee-rates/2y' \ - '/api/v1/mining/blocks/fee-rates/3y' \ - '/api/v1/mining/blocks/fee-rates/all' \ - '/api/v1/mining/difficulty-adjustments/24h' \ - '/api/v1/mining/difficulty-adjustments/3d' \ - '/api/v1/mining/difficulty-adjustments/1w' \ - '/api/v1/mining/difficulty-adjustments/1m' \ - '/api/v1/mining/difficulty-adjustments/3m' \ - '/api/v1/mining/difficulty-adjustments/6m' \ - '/api/v1/mining/difficulty-adjustments/1y' \ - '/api/v1/mining/difficulty-adjustments/2y' \ - '/api/v1/mining/difficulty-adjustments/3y' \ - '/api/v1/mining/difficulty-adjustments/all' \ - '/api/v1/lightning/channels-geo?style=widget' \ - '/api/v1/lightning/channels-geo?style=graph' \ - '/api/v1/lightning/statistics/latest' \ - '/api/v1/lightning/statistics/1m' \ - '/api/v1/lightning/statistics/3m' \ - '/api/v1/lightning/statistics/6m' \ - '/api/v1/lightning/statistics/1y' \ - '/api/v1/lightning/statistics/2y' \ - '/api/v1/lightning/statistics/3y' \ - '/api/v1/lightning/statistics/all' \ - '/api/v1/lightning/nodes/isp-ranking' \ - '/api/v1/lightning/nodes/isp/15169,396982' `# Google` \ - '/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` \ - '/api/v1/lightning/nodes/isp/39572' `# DataWeb` \ - '/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \ - '/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \ - '/api/v1/lightning/nodes/isp/394745' `# LunaNode` \ - '/api/v1/lightning/nodes/isp/45102' `# Alibaba` \ - '/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \ - '/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \ - '/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` \ - '/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` \ - '/api/v1/lightning/nodes/isp/16276' `# OVH SAS` \ - '/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,20115,33363' `# Spectrum` \ - '/api/v1/lightning/nodes/isp/701' `# Verizon` \ - '/api/v1/lightning/nodes/isp/12876' `# Scaleway` \ - '/api/v1/lightning/nodes/isp/33915' `# Ziggo` \ - '/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` \ - '/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` \ - '/api/v1/lightning/nodes/isp/212531', `# UAB Interneto vizija` \ - '/api/v1/lightning/nodes/isp/63949', `# Linode` \ - '/api/v1/lightning/nodes/isp/51167', `# Contabo GmbH` \ - '/api/v1/lightning/nodes/countries' \ - '/api/v1/lightning/nodes/rankings' \ - '/api/v1/lightning/nodes/rankings/liquidity' \ - '/api/v1/lightning/nodes/rankings/connectivity' \ - '/api/v1/lightning/nodes/rankings/age' \ +warmSlurpURLs=( + '/' + '/mining' + '/lightning' + '/graphs' + '/docs' + '/docs/faq' + '/docs/api/rest' + '/about' +) +warmUnfurlURLs=( + '/' + '/mining' + '/lightning' +) + +warmURLs=( + '/api/v1/blocks' + '/api/v1/statistics/2h' + '/api/v1/statistics/24h' + '/api/v1/statistics/1w' + '/api/v1/statistics/1m' + '/api/v1/statistics/3m' + '/api/v1/statistics/6m' + '/api/v1/statistics/1y' + '/api/v1/statistics/2y' + '/api/v1/statistics/3y' + '/api/v1/statistics/4y' + '/api/v1/statistics/all' + '/api/v1/mining/pools/24h' + '/api/v1/mining/pools/3d' + '/api/v1/mining/pools/1w' + '/api/v1/mining/pools/1m' + '/api/v1/mining/pools/3m' + '/api/v1/mining/pools/6m' + '/api/v1/mining/pools/1y' + '/api/v1/mining/pools/2y' + '/api/v1/mining/pools/3y' + '/api/v1/mining/pools/all' + '/api/v1/mining/hashrate/3m' + '/api/v1/mining/hashrate/6m' + '/api/v1/mining/hashrate/1y' + '/api/v1/mining/hashrate/2y' + '/api/v1/mining/hashrate/3y' + '/api/v1/mining/hashrate/all' + '/api/v1/mining/hashrate/pools/3m' + '/api/v1/mining/hashrate/pools/6m' + '/api/v1/mining/hashrate/pools/1y' + '/api/v1/mining/hashrate/pools/2y' + '/api/v1/mining/hashrate/pools/3y' + '/api/v1/mining/hashrate/pools/all' + '/api/v1/mining/reward-stats/144' + '/api/v1/mining/blocks/fees/24h' + '/api/v1/mining/blocks/fees/3d' + '/api/v1/mining/blocks/fees/1w' + '/api/v1/mining/blocks/fees/1m' + '/api/v1/mining/blocks/fees/3m' + '/api/v1/mining/blocks/fees/6m' + '/api/v1/mining/blocks/fees/1y' + '/api/v1/mining/blocks/fees/2y' + '/api/v1/mining/blocks/fees/3y' + '/api/v1/mining/blocks/fees/all' + '/api/v1/mining/blocks/rewards/24h' + '/api/v1/mining/blocks/rewards/3d' + '/api/v1/mining/blocks/rewards/1w' + '/api/v1/mining/blocks/rewards/1m' + '/api/v1/mining/blocks/rewards/3m' + '/api/v1/mining/blocks/rewards/6m' + '/api/v1/mining/blocks/rewards/1y' + '/api/v1/mining/blocks/rewards/2y' + '/api/v1/mining/blocks/rewards/3y' + '/api/v1/mining/blocks/rewards/all' + '/api/v1/mining/blocks/fee-rates/24h' + '/api/v1/mining/blocks/fee-rates/3d' + '/api/v1/mining/blocks/fee-rates/1w' + '/api/v1/mining/blocks/fee-rates/1m' + '/api/v1/mining/blocks/fee-rates/3m' + '/api/v1/mining/blocks/fee-rates/6m' + '/api/v1/mining/blocks/fee-rates/1y' + '/api/v1/mining/blocks/fee-rates/2y' + '/api/v1/mining/blocks/fee-rates/3y' + '/api/v1/mining/blocks/fee-rates/all' + '/api/v1/mining/difficulty-adjustments/24h' + '/api/v1/mining/difficulty-adjustments/3d' + '/api/v1/mining/difficulty-adjustments/1w' + '/api/v1/mining/difficulty-adjustments/1m' + '/api/v1/mining/difficulty-adjustments/3m' + '/api/v1/mining/difficulty-adjustments/6m' + '/api/v1/mining/difficulty-adjustments/1y' + '/api/v1/mining/difficulty-adjustments/2y' + '/api/v1/mining/difficulty-adjustments/3y' + '/api/v1/mining/difficulty-adjustments/all' + '/api/v1/lightning/channels-geo?style=widget' + '/api/v1/lightning/channels-geo?style=graph' + '/api/v1/lightning/statistics/latest' + '/api/v1/lightning/statistics/1m' + '/api/v1/lightning/statistics/3m' + '/api/v1/lightning/statistics/6m' + '/api/v1/lightning/statistics/1y' + '/api/v1/lightning/statistics/2y' + '/api/v1/lightning/statistics/3y' + '/api/v1/lightning/statistics/all' + '/api/v1/lightning/nodes/isp-ranking' + '/api/v1/lightning/nodes/isp/15169,396982' `# Google` + '/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` + '/api/v1/lightning/nodes/isp/39572' `# DataWeb` + '/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` + '/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` + '/api/v1/lightning/nodes/isp/394745' `# LunaNode` + '/api/v1/lightning/nodes/isp/45102' `# Alibaba` + '/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` + '/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` + '/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` + '/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` + '/api/v1/lightning/nodes/isp/16276' `# OVH SAS` + '/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,2$delay,33363' `# Spectrum` + '/api/v1/lightning/nodes/isp/701' `# Verizon` + '/api/v1/lightning/nodes/isp/12876' `# Scaleway` + '/api/v1/lightning/nodes/isp/33915' `# Ziggo` + '/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` + '/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` + '/api/v1/lightning/nodes/isp/212531' `# UAB Interneto vizija` + '/api/v1/lightning/nodes/isp/63949' `# Linode` + '/api/v1/lightning/nodes/isp/51167' `# Contabo GmbH` + '/api/v1/lightning/nodes/countries' + '/api/v1/lightning/nodes/rankings' + '/api/v1/lightning/nodes/rankings/liquidity' + '/api/v1/lightning/nodes/rankings/connectivity' + '/api/v1/lightning/nodes/rankings/age' +) + +while true +do + echo "starting warm cache cycle..." + + for url in $warmSlurpURLs + do + warmSlurp "https://${hostname}${url}" + sleep $delay # delay between queries to not DoS unfurler + done + + for url in $warmUnfurlURLs + do + warmUnfurl "https://${hostname}${url}" + sleep $delay # delay between queries to not DoS unfurler + done + + for url in $warmURLs do warm "https://${hostname}${url}" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb done for slug in $slugs do warm "https://${hostname}/api/v1/mining/pool/${slug}" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb done + + sleep 1 done From 8f2a2d1c8187e5db5787aa74083667189cf2af33 Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 16 Aug 2023 04:03:17 +0900 Subject: [PATCH 009/152] ops: Add delay before warm cache loop begins --- production/nginx-cache-warmer | 3 +++ 1 file changed, 3 insertions(+) diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index fe3587c93..f02091747 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -153,6 +153,9 @@ warmURLs=( '/api/v1/lightning/nodes/rankings/age' ) +echo "waiting for mempool backend to start..." +sleep 60 + while true do echo "starting warm cache cycle..." From b1e1601ccf661913f928c7435c8bd568922019e1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 16 Aug 2023 21:55:03 +0900 Subject: [PATCH 010/152] Switch unfurl proxy header to url prefix --- unfurler/src/index.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index ae751d1f5..69882579f 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -109,11 +109,15 @@ class Server { setUpRoutes() { if (puppeteerEnabled) { + this.app.get('/unfurl/render*', async (req, res) => { return this.renderPreview(req, res) }) this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) } else { + this.app.get('/unfurl/render*', async (req, res) => { return this.renderDisabled(req, res) }) this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) }) } - this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) + this.app.get('/unfurl*', (req, res) => { return this.renderHTML(req, res, true) }) + this.app.get('/slurp*', (req, res) => { return this.renderHTML(req, res, false) }) + this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) }) } async clusterTask({ page, data: { url, path, action } }) { @@ -240,7 +244,7 @@ class Server { } } - async renderHTML(req, res) { + async renderHTML(req, res, unfurl: boolean = false) { // drop requests for static files const rawPath = req.params[0]; const match = rawPath.match(/\.[\w]+$/); @@ -250,7 +254,7 @@ class Server { || rawPath.startsWith('/api/v1/translators/images') || rawPath.startsWith('/resources/profile') ) { - if (isPreviewCrawler(req)) { + if (unfurl) { res.status(404).send(); return; } else { @@ -265,7 +269,7 @@ class Server { let result = ''; try { - if (isPreviewCrawler(req)) { + if (unfurl) { result = await this.renderUnfurlMeta(rawPath); } else { result = await this.renderSEOPage(rawPath); @@ -348,7 +352,3 @@ function capitalize(str) { return str; } } - -function isPreviewCrawler(req: Request): boolean { - return req?.header('X-Unfurl-Type') === 'preview'; -} From 9aedccdb320f7619c80c7622858b7638d27f02db Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Wed, 16 Aug 2023 16:07:27 -0700 Subject: [PATCH 011/152] Remove trailing comma on the Docker JSON --- docker/backend/mempool-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index e283d1171..aa084133f 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -52,7 +52,7 @@ "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__, - "FALLBACK": __ESPLORA_FALLBACK__, + "FALLBACK": __ESPLORA_FALLBACK__ }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", From 27797b8f1d30582778553f88cf9c78a9867b130c Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Wed, 16 Aug 2023 16:08:01 -0700 Subject: [PATCH 012/152] Add missing ESPLORA FALLBACK vars to the start script --- docker/backend/start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 681872681..2e293ce34 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -53,6 +53,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} __ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"} __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} +__ESPLORA_FALLBACK__=${ESPLORA_FALLBACK:=[]} # SECOND_CORE_RPC __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1} @@ -192,6 +193,7 @@ sed -i "s!__ELECTRUM_TLS_ENABLED__!${__ELECTRUM_TLS_ENABLED__}!g" mempool-config sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json +sed -i "s!__ESPLORA_FALLBACK__!${__ESPLORA_FALLBACK__}!g" mempool-config.json sed -i "s!__SECOND_CORE_RPC_HOST__!${__SECOND_CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__SECOND_CORE_RPC_PORT__!${__SECOND_CORE_RPC_PORT__}!g" mempool-config.json From 44d2ea126358ef58cc2a6626be866b332308852a Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Wed, 16 Aug 2023 16:08:45 -0700 Subject: [PATCH 013/152] Fix Docker config unit test to work with array values --- backend/src/__tests__/config.test.ts | 37 +++++++++++++++------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 2370fe7a1..5a7c4ba4b 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -191,30 +191,31 @@ describe('Mempool Backend Config', () => { } continue; } - switch (typeof value) { - case 'object': { - if (Array.isArray(value)) { - continue; - } else { - parseJson(value, key); - } - break; - } - default: { + + if (root) { //The flattened string, i.e, __MEMPOOL_ENABLED__ const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; //The string used as the environment variable, i.e, MEMPOOL_ENABLED const envVarStr = `${root ? root : ''}_${key}`; + let defaultEntry; //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)} - const defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; - - if (process.env.CI) { - console.log(`looking for ${defaultEntry} in the start.sh script`); + if (Array.isArray(value)) { + defaultEntry = `${replaceStr}=\$\{${envVarStr}:=\[]}`; + if (process.env.CI) { + console.log(`looking for ${defaultEntry} in the start.sh script`); + } + //Regex matching does not work with the array values + expect(startSh).toContain(defaultEntry); + } else { + defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; + if (process.env.CI) { + console.log(`looking for ${defaultEntry} in the start.sh script`); + } + const re = new RegExp(defaultEntry); + expect(startSh).toMatch(re); } - const re = new RegExp(defaultEntry); - expect(startSh).toMatch(re); //The string that actually replaces the values in the config file const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json'; @@ -222,11 +223,13 @@ describe('Mempool Backend Config', () => { console.log(`looking for ${sedStr} in the start.sh script`); } expect(startSh).toContain(sedStr); - break; } + else { + parseJson(value, key); } } } + parseJson(fixture); }); }); From 004208e9c87034b43b3c4d9782362f44d6934293 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Wed, 16 Aug 2023 16:16:04 -0700 Subject: [PATCH 014/152] Fix linting --- backend/src/__tests__/config.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 5a7c4ba4b..8097a2465 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -202,7 +202,7 @@ describe('Mempool Backend Config', () => { let defaultEntry; //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)} if (Array.isArray(value)) { - defaultEntry = `${replaceStr}=\$\{${envVarStr}:=\[]}`; + defaultEntry = `${replaceStr}=\${${envVarStr}:=[]}`; if (process.env.CI) { console.log(`looking for ${defaultEntry} in the start.sh script`); } From 2c9e20dd87d8b64fa1ef49f21e19a076f8df1bda Mon Sep 17 00:00:00 2001 From: wiz Date: Thu, 17 Aug 2023 18:38:12 +0900 Subject: [PATCH 015/152] ops: Always start xorg for unfurler even if no GPU detected --- production/mempool-start-all | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/production/mempool-start-all b/production/mempool-start-all index 306156660..7efbf7dd6 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -10,12 +10,9 @@ for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-li screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done' done -# only start xorg if GPU present -if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then - export DISPLAY=:0 - screen -dmS x startx - sleep 3 -fi +export DISPLAY=:0 +screen -dmS x startx +sleep 3 # start unfurlers for each frontend for site in mainnet liquid bisq;do From ab911d5c9eca061bbdd85a71eceb919c3a95deb5 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 17 Aug 2023 14:28:33 +0200 Subject: [PATCH 016/152] [tx] fix eta css with accelerate button --- .../components/transaction/transaction.component.html | 10 ++++++---- .../components/transaction/transaction.component.ts | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 133bcaa1d..ddbfd0bb3 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -99,9 +99,10 @@ - + In several hours (or more) - Accelerate + + Accelerate @@ -109,9 +110,10 @@ - + - Accelerate + + Accelerate diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index f1f3850e4..d12be2084 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -82,6 +82,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { blockConversion: Price; tooltipPosition: { x: number, y: number }; isMobile: boolean; + acceleratorAvailable: boolean = false; featuresEnabled: boolean; segwitEnabled: boolean; @@ -107,6 +108,8 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { ) {} ngOnInit() { + this.acceleratorAvailable = this.stateService.env.OFFICIAL_MEMPOOL_SPACE && this.stateService.env.ACCELERATOR && this.stateService.network === ''; + this.websocketService.want(['blocks', 'mempool-blocks']); this.stateService.networkChanged$.subscribe( (network) => (this.network = network) From 23b871631a83e894b9d011cc3b17493b4f973b6e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 17 Aug 2023 18:51:39 +0200 Subject: [PATCH 017/152] [menu] write menu component with hardcoded values --- .../master-page/master-page.component.html | 4 +- .../app/components/menu/menu.component.html | 17 +++++++ .../app/components/menu/menu.component.scss | 46 +++++++++++++++++ .../src/app/components/menu/menu.component.ts | 23 +++++++++ .../src/app/interfaces/services.interface.ts | 13 +++++ frontend/src/app/services/api.service.ts | 50 ++++++++++++++++++- frontend/src/app/shared/shared.module.ts | 20 +++++++- 7 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 frontend/src/app/components/menu/menu.component.html create mode 100644 frontend/src/app/components/menu/menu.component.scss create mode 100644 frontend/src/app/components/menu/menu.component.ts create mode 100644 frontend/src/app/interfaces/services.interface.ts diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index f12fbc960..77212b1a3 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -1,5 +1,7 @@ + + -
+