machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into simon/da-api-handle-error

Browse Source
This commit is contained in:
wiz
2022-08-30 19:35:57 +02:00
committed by GitHub
parent 936921781e abb2ce5146
commit d9320574d8
96 changed files with 1718 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
backend/src/api/common.ts
View File

@@ -228,34 +228,75 @@ export class Common {
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
static findSocketNetwork(addr: string): {network: string | null, url: string} {
let network: string | null = null;
let url = addr.split('://')[1];
if (config.LIGHTNING.BACKEND === 'cln') {
network = socket.network;
} else if (config.LIGHTNING.BACKEND === 'lnd') {
if (socket.addr.indexOf('onion') !== -1) {
if (socket.addr.split('.')[0].length >= 56) {
network = 'torv3';
} else {
network = 'torv2';
}
} else if (socket.addr.indexOf('i2p') !== -1) {
network = 'i2p';
if (!url) {
return {
network: null,
url: addr,
};
}
if (addr.indexOf('onion') !== -1) {
if (url.split('.')[0].length >= 56) {
network = 'torv3';
} else {
const ipv = isIP(socket.addr.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
} else if (ipv === 6) {
network = 'ipv6';
}
network = 'torv2';
}
} else if (addr.indexOf('i2p') !== -1) {
network = 'i2p';
} else if (addr.indexOf('ipv4') !== -1) {
const ipv = isIP(url.split(':')[0]);
if (ipv === 4) {
network = 'ipv4';
} else {
return {
network: null,
url: addr,
};
}
} else if (addr.indexOf('ipv6') !== -1) {
url = url.split('[')[1].split(']')[0];
const ipv = isIP(url);
if (ipv === 6) {
const parts = addr.split(':');
network = 'ipv6';
url = `[${url}]:${parts[parts.length - 1]}`;
} else {
return {
network: null,
url: addr,
};
}
} else {
return {
network: null,
url: addr,
};
}
return {
publicKey: publicKey,
network: network,
addr: socket.addr,
url: url,
};
}
static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
if (config.LIGHTNING.BACKEND === 'cln') {
return {
publicKey: publicKey,
network: socket.network,
addr: socket.addr,
};
} else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
const formatted = this.findSocketNetwork(socket.addr);
return {
publicKey: publicKey,
network: formatted.network,
addr: formatted.url,
};
}
}
}

33
backend/src/api/explorer/channels.routes.ts
View File

@@ -70,7 +70,7 @@ class ChannelsRoutes {
}
}
private async $getChannelsByTransactionIds(req: Request, res: Response) {
private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
try {
if (!Array.isArray(req.query.txId)) {
res.status(400).send('Not an array');
@@ -83,27 +83,26 @@ class ChannelsRoutes {
}
}
const channels = await channelsApi.$getChannelsByTransactionId(txIds);
const inputs: any[] = [];
const outputs: any[] = [];
const result: any[] = [];
for (const txid of txIds) {
const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid);
if (foundChannelInputs) {
inputs.push(foundChannelInputs);
} else {
inputs.push(null);
const inputs: any = {};
const outputs: any = {};
// Assuming that we only have one lightning close input in each transaction. This may not be true in the future
const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
if (foundChannelsFromInput) {
inputs[0] = foundChannelsFromInput;
}
const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid);
if (foundChannelOutputs) {
outputs.push(foundChannelOutputs);
} else {
outputs.push(null);
const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
for (const output of foundChannelsFromOutputs) {
outputs[output.transaction_vout] = output;
}
result.push({
inputs,
outputs,
});
}
res.json({
inputs: inputs,
outputs: outputs,
});
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}

13
backend/src/api/explorer/nodes.api.ts
View File

@@ -412,6 +412,7 @@ class NodesApi {
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM nodes
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
@@ -502,6 +503,18 @@ class NodesApi {
}
}
/**
* Update node sockets
*/
public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
try {
await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
} catch (e) {
logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
}
}
/**
* Set all nodes not in `nodesPubkeys` as inactive (status = 0)
*/

2
backend/src/api/explorer/statistics.api.ts
View File

@@ -27,7 +27,7 @@ class StatisticsApi {
public async $getLatestStatistics(): Promise<any> {
try {
const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`);
const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
return {
latest: rows[0],
previous: rows2[0],

2
backend/src/logger.ts
View File

@@ -74,7 +74,7 @@ class Logger {
private getNetwork(): string {
if (config.LIGHTNING.ENABLED) {
return 'lightning';
return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`;
}
if (config.BISQ.ENABLED) {
return 'bisq';

10
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@@ -4,6 +4,7 @@ import nodesApi from '../../../api/explorer/nodes.api';
import config from '../../../config';
import DB from '../../../database';
import logger from '../../../logger';
import * as IPCheck from '../../../utils/ipcheck.js';
export async function $lookupNodeLocation(): Promise<void> {
let loggerTimer = new Date().getTime() / 1000;
@@ -27,6 +28,11 @@ export async function $lookupNodeLocation(): Promise<void> {
const asn = lookupAsn.get(ip);
const isp = lookupIsp.get(ip);
let asOverwrite: number | null = null;
if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
asOverwrite = 394745;
}
if (city && (asn || isp)) {
const query = `
UPDATE nodes SET
@@ -41,7 +47,7 @@ export async function $lookupNodeLocation(): Promise<void> {
`;
const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number,
asOverwrite ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
city.city?.geoname_id,
city.country?.geoname_id,
city.subdivisions ? city.subdivisions[0].geoname_id : null,
@@ -91,7 +97,7 @@ export async function $lookupNodeLocation(): Promise<void> {
if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
[isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
[asOverwrite ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
}
}

86
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@@ -8,6 +8,7 @@ import { isIP } from 'net';
import { Common } from '../../../api/common';
import channelsApi from '../../../api/explorer/channels.api';
import nodesApi from '../../../api/explorer/nodes.api';
import { ResultSetHeader } from 'mysql2';
const fsPromises = promises;
@@ -19,7 +20,12 @@ class LightningStatsImporter {
logger.info('Caching funding txs for currently existing channels');
await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
return;
}
await this.$importHistoricalLightningStats();
await this.$cleanupIncorrectSnapshot();
}
/**
@@ -51,6 +57,8 @@ class LightningStatsImporter {
features: node.features,
});
nodesInDb[node.pub_key] = node;
} else {
await nodesApi.$updateNodeSockets(node.pub_key, node.addresses);
}
let hasOnion = false;
@@ -363,10 +371,16 @@ class LightningStatsImporter {
graph = JSON.parse(fileContent);
graph = await this.cleanupTopology(graph);
} catch (e) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
continue;
}
if (this.isIncorrectSnapshot(timestamp, graph)) {
logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
++totalProcessed;
continue;
}
if (!logStarted) {
logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
logStarted = true;
@@ -397,7 +411,7 @@ class LightningStatsImporter {
}
}
async cleanupTopology(graph) {
cleanupTopology(graph): ILightningApi.NetworkGraph {
const newGraph = {
nodes: <ILightningApi.Node[]>[],
edges: <ILightningApi.Channel[]>[],
@@ -407,9 +421,10 @@ class LightningStatsImporter {
const addressesParts = (node.addresses ?? '').split(',');
const addresses: any[] = [];
for (const address of addressesParts) {
const formatted = Common.findSocketNetwork(address);
addresses.push({
network: '',
addr: address
network: formatted.network,
addr: formatted.url
});
}
@@ -456,6 +471,69 @@ class LightningStatsImporter {
return newGraph;
}
private isIncorrectSnapshot(timestamp, graph): boolean {
if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
return true;
}
if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
return true;
}
if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
return true;
}
if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
return true;
}
if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
return true;
}
if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
return true;
}
if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
return true;
}
if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
return true;
}
if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
return true;
}
return false;
}
private async $cleanupIncorrectSnapshot(): Promise<void> {
// We do not run this one automatically because those stats are not supposed to be inserted in the first
// place, but I write them here to remind us we manually run those queries
// DELETE FROM lightning_stats
// WHERE (
// UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
// UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
// UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
// UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
// UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
// UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
// UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
// UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
// UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
// )
// DELETE FROM node_stats
// WHERE (
// UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
// UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
// UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
// UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
// UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
// UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
// UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
// UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
// UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000
// )
}
}
export default new LightningStatsImporter;

119
backend/src/utils/ipcheck.js Normal file
View File

@@ -0,0 +1,119 @@
var net = require('net');
var IPCheck = module.exports = function(input) {
var self = this;
if (!(self instanceof IPCheck)) {
return new IPCheck(input);
}
self.input = input;
self.parse();
};
IPCheck.prototype.parse = function() {
var self = this;
if (!self.input || typeof self.input !== 'string') return self.valid = false;
var ip;
var pos = self.input.lastIndexOf('/');
if (pos !== -1) {
ip = self.input.substring(0, pos);
self.mask = +self.input.substring(pos + 1);
} else {
ip = self.input;
self.mask = null;
}
self.ipv = net.isIP(ip);
self.valid = !!self.ipv && !isNaN(self.mask);
if (!self.valid) return;
// default mask = 32 for ipv4 and 128 for ipv6
if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
if (self.ipv === 4) {
// difference between ipv4 and ipv6 masks
self.mask += 96;
}
if (self.mask < 0 || self.mask > 128) {
self.valid = false;
return;
}
self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
if(self.ipv === 4){
self.parseIPv4(ip);
}else{
self.parseIPv6(ip);
}
};
IPCheck.prototype.parseIPv4 = function(ip) {
var self = this;
// ipv4 addresses live under ::ffff:0:0
self.address[10] = self.address[11] = 0xff;
var octets = ip.split('.');
for (var i = 0; i < 4; i++) {
self.address[i + 12] = parseInt(octets[i], 10);
}
};
var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
IPCheck.prototype.parseIPv6 = function(ip) {
var self = this;
var transitionalMatch = V6_TRANSITIONAL.exec(ip);
if(transitionalMatch){
self.parseIPv4(transitionalMatch[1]);
return;
}
var bits = ip.split(':');
if (bits.length < 8) {
ip = ip.replace('::', Array(11 - bits.length).join(':'));
bits = ip.split(':');
}
var j = 0;
for (var i = 0; i < bits.length; i += 1) {
var x = bits[i] ? parseInt(bits[i], 16) : 0;
self.address[j++] = x >> 8;
self.address[j++] = x & 0xff;
}
};
IPCheck.prototype.match = function(cidr) {
var self = this;
if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
if (!self.valid || !cidr.valid) return false;
var mask = cidr.mask;
var i = 0;
while (mask >= 8) {
if (self.address[i] !== cidr.address[i]) return false;
i++;
mask -= 8;
}
var shift = 8 - mask;
return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
};
IPCheck.match = function(ip, cidr) {
ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
return ip.match(cidr);
};