machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/difficulty-api

Browse Source
This commit is contained in:
wiz
2022-08-24 20:01:52 +09:00
committed by GitHub
parent 3a4982c5e6 b121e46bcc
commit 00bd61e1d3
63 changed files with 1125 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 38;
private static currentVersion = 39;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -248,7 +248,6 @@ class DatabaseMigration {
}
if (databaseSchemaVersion < 25 && isBitcoin === true) {
await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
@@ -338,6 +337,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
}
if (databaseSchemaVersion < 39 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
}
}
/**

103
backend/src/api/explorer/channels.api.ts
View File

@@ -17,32 +17,60 @@ class ChannelsApi {
}
}
public async $getAllChannelsGeo(publicKey?: string): Promise<any[]> {
public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
try {
let select: string;
if (style === 'widget') {
select = `
nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
`;
} else {
select = `
nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
`;
}
const params: string[] = [];
let query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude,
channels.capacity
FROM channels
JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
let query = `SELECT ${select}
FROM channels
JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
`;
if (publicKey !== undefined) {
query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
params.push(publicKey);
params.push(publicKey);
} else {
query += ` AND channels.capacity > 1000000
GROUP BY nodes_1.public_key, nodes_2.public_key
ORDER BY channels.capacity DESC
LIMIT 10000
`;
}
const [rows]: any = await DB.query(query, params);
return rows.map((row) => [
row.node1_public_key, row.node1_alias, row.node1_longitude, row.node1_latitude,
row.node2_public_key, row.node2_alias, row.node2_longitude, row.node2_latitude,
row.capacity]);
return rows.map((row) => {
if (style === 'widget') {
return [
row.node1_longitude, row.node1_latitude,
row.node2_longitude, row.node2_latitude,
];
} else {
return [
row.node1_public_key, row.node1_alias,
row.node1_longitude, row.node1_latitude,
row.node2_public_key, row.node2_alias,
row.node2_longitude, row.node2_latitude,
];
}
});
} catch (e) {
logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
throw e;
@@ -61,9 +89,14 @@ class ChannelsApi {
}
}
public async $getChannelsByStatus(status: number): Promise<any[]> {
public async $getChannelsByStatus(status: number | number[]): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE status = ?`;
let query: string;
if (Array.isArray(status)) {
query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`;
} else {
query = `SELECT * FROM channels WHERE status = ?`;
}
const [rows]: any = await DB.query(query, [status]);
return rows;
} catch (e) {
@@ -212,41 +245,53 @@ class ChannelsApi {
let channelStatusFilter;
if (status === 'open') {
channelStatusFilter = '< 2';
} else if (status === 'active') {
channelStatusFilter = '= 1';
} else if (status === 'closed') {
channelStatusFilter = '= 2';
} else {
throw new Error('getChannelsForNode: Invalid status requested');
}
// Channels originating from node
let query = `
SELECT node2.alias, node2.public_key, channels.status, channels.node1_fee_rate,
channels.capacity, channels.short_id, channels.id
SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
channels.status, channels.node1_fee_rate,
channels.capacity, channels.short_id, channels.id, channels.closing_reason
FROM channels
JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
`;
const [channelsFromNode]: any = await DB.query(query, [public_key, index, length]);
const [channelsFromNode]: any = await DB.query(query, [public_key]);
// Channels incoming to node
query = `
SELECT node1.alias, node1.public_key, channels.status, channels.node2_fee_rate,
channels.capacity, channels.short_id, channels.id
SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
channels.status, channels.node2_fee_rate,
channels.capacity, channels.short_id, channels.id, channels.closing_reason
FROM channels
JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
`;
const [channelsToNode]: any = await DB.query(query, [public_key, index, length]);
const [channelsToNode]: any = await DB.query(query, [public_key]);
let allChannels = channelsFromNode.concat(channelsToNode);
allChannels.sort((a, b) => {
return b.capacity - a.capacity;
});
allChannels = allChannels.slice(index, index + length);
if (index >= 0) {
allChannels = allChannels.slice(index, index + length);
} else if (index === -1) { // Node channels tree chart
allChannels = allChannels.slice(0, 1000);
}
const channels: any[] = []
for (const row of allChannels) {
const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
channels.push({
status: row.status,
closing_reason: row.closing_reason,
capacity: row.capacity ?? 0,
short_id: row.short_id,
id: row.id,
@@ -337,7 +382,7 @@ class ChannelsApi {
/**
* Save or update a channel present in the graph
*/
public async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
const [ txid, vout ] = channel.chan_point.split(':');
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
@@ -369,11 +414,11 @@ class ChannelsApi {
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
status = ${status},
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,

6
backend/src/api/explorer/channels.routes.ts
View File

@@ -102,7 +102,11 @@ class ChannelsRoutes {
private async $getAllChannelsGeo(req: Request, res: Response) {
try {
const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey);
const style: string = typeof req.query.style === 'string' ? req.query.style : '';
const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);

50
backend/src/api/explorer/nodes.api.ts
View File

@@ -169,7 +169,7 @@ class NodesApi {
let query: string;
if (full === false) {
query = `
SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
node_stats.channels
FROM node_stats
JOIN nodes ON nodes.public_key = node_stats.public_key
@@ -259,9 +259,10 @@ class NodesApi {
public async $searchNodeByPublicKeyOrAlias(search: string) {
try {
const searchStripped = search.replace('%', '') + '%';
const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR nodes.alias LIKE ? GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
const publicKeySearch = search.replace('%', '') + '%';
const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR MATCH nodes.alias_search AGAINST (? IN BOOLEAN MODE) GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
return rows;
} catch (e) {
logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
@@ -296,19 +297,24 @@ class NodesApi {
if (!ispList[isp1]) {
ispList[isp1] = {
id: channel.isp1ID,
id: channel.isp1ID.toString(),
capacity: 0,
channels: 0,
nodes: {},
};
} else if (ispList[isp1].id.indexOf(channel.isp1ID) === -1) {
ispList[isp1].id += ',' + channel.isp1ID.toString();
}
if (!ispList[isp2]) {
ispList[isp2] = {
id: channel.isp2ID,
id: channel.isp2ID.toString(),
capacity: 0,
channels: 0,
nodes: {},
};
} else if (ispList[isp2].id.indexOf(channel.isp2ID) === -1) {
ispList[isp2].id += ',' + channel.isp2ID.toString();
}
ispList[isp1].capacity += channel.capacity;
@@ -385,9 +391,10 @@ class NodesApi {
public async $getNodesPerCountry(countryId: string) {
try {
const query = `
SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city
SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM node_stats
JOIN (
SELECT public_key, MAX(added) as last_added
@@ -395,15 +402,19 @@ class NodesApi {
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
WHERE geo_names_country.id = ?
ORDER BY capacity DESC
`;
const [rows]: any = await DB.query(query, [countryId]);
for (let i = 0; i < rows.length; ++i) {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);
rows[i].subdivision = JSON.parse(rows[i].subdivision);
}
return rows;
} catch (e) {
@@ -417,7 +428,8 @@ class NodesApi {
const query = `
SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country
geo_names_city.names as city, geo_names_country.names as country,
geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
FROM node_stats
JOIN (
SELECT public_key, MAX(added) as last_added
@@ -425,8 +437,10 @@ class NodesApi {
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
WHERE nodes.as_number IN (?)
ORDER BY capacity DESC
`;
@@ -435,6 +449,7 @@ class NodesApi {
for (let i = 0; i < rows.length; ++i) {
rows[i].country = JSON.parse(rows[i].country);
rows[i].city = JSON.parse(rows[i].city);
rows[i].subdivision = JSON.parse(rows[i].subdivision);
}
return rows;
} catch (e) {
@@ -487,21 +502,24 @@ class NodesApi {
first_seen,
updated_at,
alias,
alias_search,
color,
sockets,
status
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?, status = 1`;
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
await DB.query(query, [
node.pub_key,
node.last_update,
node.alias,
this.aliasToSearchText(node.alias),
node.color,
sockets,
node.last_update,
node.alias,
this.aliasToSearchText(node.alias),
node.color,
sockets,
]);
@@ -535,6 +553,10 @@ class NodesApi {
logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
private aliasToSearchText(str: string): string {
return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, '');
}
}
export default new NodesApi();

4
backend/src/api/explorer/nodes.routes.ts
View File

@@ -15,8 +15,8 @@ class NodesRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/capacity', this.$getTopNodesByCapacity)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/channels', this.$getTopNodesByChannels)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/liquidity', this.$getTopNodesByCapacity)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)