From 7fd213a0fa6920e52ae93d3d29bcab8729225232 Mon Sep 17 00:00:00 2001 From: kenny-io Date: Fri, 18 Oct 2024 00:17:13 +0400 Subject: [PATCH 1/3] add nodelist and network account methods --- package-lock.json | 18 ++++++++ package.json | 1 + src/api.ts | 95 ++++++++++++++++++++++++++++++++++++++++ src/config.ts | 6 +-- src/external/Archiver.ts | 25 +++++++++++ src/utils.ts | 44 ++++++++++++++++++- 6 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 src/external/Archiver.ts diff --git a/package-lock.json b/package-lock.json index 0f4af65e..e0c052e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "axios": "1.6.1", "better-sqlite3": "7.6.2", "body-parser": "1.19.0", + "cache-memory": "^3.0.3", "connect": "3.7.0", "cookie-parser": "1.4.6", "cors": "2.8.5", @@ -1973,6 +1974,15 @@ "node": ">= 0.8" } }, + "node_modules/cache-memory": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/cache-memory/-/cache-memory-3.0.3.tgz", + "integrity": "sha512-xRSL5UgIrlXaNRKKJsGMX4sGLV2Qamc6N/ABBJy1Cqx4IQl/V6JCgdgZc42z4YAEm4SqhSoPwkZNcV4N6USq0w==", + "dependencies": { + "clone": "^2.1.2", + "debug": "^4.3.4" + } + }, "node_modules/cacheable-lookup": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", @@ -2177,6 +2187,14 @@ "node": ">= 10" } }, + "node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/clone-response": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", diff --git a/package.json b/package.json index dad97f61..963cbe3f 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "axios": "1.6.1", "better-sqlite3": "7.6.2", "body-parser": "1.19.0", + "cache-memory": "^3.0.3", "connect": "3.7.0", "cookie-parser": "1.4.6", "cors": "2.8.5", diff --git a/src/api.ts b/src/api.ts index 7c9479fd..f6cc452e 100755 --- a/src/api.ts +++ b/src/api.ts @@ -46,6 +46,7 @@ import { bytesToHex, toBytes } from '@ethereumjs/util' import { RLP } from '@ethereumjs/rlp' import { nestedCountersInstance } from './utils/nestedCounters' import { trySpendServicePoints } from './utils/servicePoints' +import { archiverAPI } from './external/Archiver' export const verbose = config.verbose export const firstLineLogs = config.firstLineLogs @@ -3783,6 +3784,100 @@ export const methods = { countFailedResponse(api_name, 'exception in /contract/accesslist') } }, + + shardeum_getNodeList: async function (args: RequestParamsLike, callback: JSONRPCCallbackTypePlain) { + const api_name = 'shardeum_getNodeList' + const ticket = crypto + .createHash('sha1') + .update(api_name + Math.random() + Date.now()) + .digest('hex') + logEventEmitter.emit('fn_start', ticket, api_name, performance.now()) + + try { + if (!Array.isArray(args) || args.length > 1 || (args.length === 1 && typeof args[0] !== 'object')) { + callback({ code: -32602, message: 'Invalid params: Expected an object or no parameters.' }, null); + countFailedResponse(api_name, 'Invalid params'); + return; + } + + const params = args[0] || {}; + const page = Math.max(1, parseInt(params.page) || 1); + const limit = Math.min(1000, Math.max(1, parseInt(params.limit) || 100)); + const nodeListResult = await archiverAPI.getNodeList(page, limit); + + logEventEmitter.emit('fn_end', ticket, { success: true }, performance.now()) + callback(null, nodeListResult); + countSuccessResponse(api_name, 'success', 'archiver'); + } catch (error: any) { + logEventEmitter.emit('fn_end', ticket, { success: false, error: error.message }, performance.now()) + callback(error, null); + } + }, + + shardeum_getNetworkAccount: async function (args: RequestParamsLike, callback: JSONRPCCallbackTypePlain) { + const api_name = 'shardeum_getNetworkAccount'; + nestedCountersInstance.countEvent('endpoint', api_name); + + const ticket = crypto + .createHash('sha1') + .update(api_name + Math.random() + Date.now()) + .digest('hex'); + + logEventEmitter.emit('fn_start', ticket, api_name, performance.now()); + + try { + const networkAccountData = await archiverAPI.getNetworkAccount(); + + if (networkAccountData && networkAccountData.networkAccount && networkAccountData.networkAccount.data) { + const current = networkAccountData.networkAccount.data.current; + const archiver = current.archiver || {}; + + // transform the response to match the expected structure + const response = { + activeVersion: archiver.activeVersion || current.activeVersion, + latestVersion: archiver.latestVersion || current.latestVersion, + minVersion: archiver.minVersion || current.minVersion, + certCycleDuration: current.certCycleDuration, + maintenanceFee: current.maintenanceFee, + maintenanceInterval: current.maintenanceInterval, + penalty: { + amount: current.nodePenaltyUsd.value, + currency: "shm" + }, + slashing: { + leftNetworkEarlyPenaltyPercent: 0.2, + nodeRefutedPenaltyPercent: 0.2, + syncTimeoutPenaltyPercent: 0.2 + }, + reward: { + amount: current.nodeRewardAmountUsd.value, + currency: "shm", + nodeRewardInterval: current.nodeRewardInterval, + }, + requiredStake: { + amount: current.stakeRequiredUsd.value, + currency: "shm" + }, + restakeCooldown: current.restakeCooldown, + timestamp: networkAccountData.networkAccount.timestamp + }; + + logEventEmitter.emit('fn_end', ticket, { success: true }, performance.now()); + callback(null, response); + countSuccessResponse(api_name, 'success', 'archiver'); + } else { + logEventEmitter.emit('fn_end', ticket, { success: false }, performance.now()); + callback({ code: -32000, message: 'Network account data not found or invalid' }, null); + countFailedResponse(api_name, 'Network account data not found or invalid'); + } + } catch (error: any) { + console.error('Error fetching network account data:', error); + logEventEmitter.emit('fn_end', ticket, { success: false }, performance.now()); + callback({ code: -32000, message: 'Failed to fetch network account data', data: error.message }, null); + countFailedResponse(api_name, 'Failed to fetch network account data'); + } + }, + eth_subscribe: async function (args: RequestParamsLike, callback: JSONRPCCallbackTypePlain) { const api_name = 'eth_subscribe' nestedCountersInstance.countEvent('endpoint', api_name) diff --git a/src/config.ts b/src/config.ts index 0b22a216..9825456c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -183,12 +183,12 @@ export const CONFIG: Config = { isRemoteLocalNetwork: false, nodeExternalIpForRemoteLocalNetwork: '127.0.0.1', collectorSourcing: { - enabled: false, + enabled: true, collectorApiServerUrl: 'http://0.0.0.0:6001', }, serviceValidatorSourcing: { - enabled: false, - serviceValidatorUrl: 'http://0.0.0.0:9001', + enabled: true, + serviceValidatorUrl: 'http://0.0.0.0:7001', }, ServicePointsPerSecond: 200, diff --git a/src/external/Archiver.ts b/src/external/Archiver.ts new file mode 100644 index 00000000..37df3412 --- /dev/null +++ b/src/external/Archiver.ts @@ -0,0 +1,25 @@ +import { getNodeList, getNetworkAccount } from '../utils'; +import axios from 'axios'; + +export class Archiver { + constructor() {} + + async getNodeList(page: number, limit: number): Promise { + try { + return getNodeList(page, limit); + } catch (error) { + console.error('Error fetching node list:', error); + throw error; + } + } + async getNetworkAccount(): Promise { + try { + return getNetworkAccount(); + } catch (error) { + console.error('Error fetching network account:', error); + throw error; + } + } +} + +export const archiverAPI = new Archiver(); \ No newline at end of file diff --git a/src/utils.ts b/src/utils.ts index ffa63e69..a23fb0e8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -32,7 +32,7 @@ import { } from './types' import Sntp from '@hapi/sntp' import { randomBytes, createHash } from 'crypto' - +import cacheMemory from 'cache-memory' crypto.init('69fa4195670576c0160d660c3be36556ff8d504725be8a59b5a96509e0c994bc') const existingArchivers: Archiver[] = [] @@ -44,6 +44,22 @@ export const node = { port: 9001, } +const NODE_LIST_CACHE_TTL = 15 // 15 seconds +const NODE_LIST_CACHE_KEY = 'nodeList' + +const NETWORK_ACCOUNT_CACHE_TTL = 5 // 5 seconds +const NETWORK_ACCOUNT_CACHE_KEY = 'networkAccount' + +const nodeListCache = cacheMemory + .ttl(NODE_LIST_CACHE_TTL) + .storeUndefinedObjects(false) + .create({ id: 'nodeListCache' }); + + const networkAccountCache = cacheMemory + .ttl(NETWORK_ACCOUNT_CACHE_TTL) + .storeUndefinedObjects(false) + .create({ id: 'networkAccountCache' }); + let rotationEdgeToAvoid = 0 const badNodesMap: Map = new Map() @@ -183,6 +199,32 @@ export async function updateNodeList(tryInfinate = false): Promise { } } console.timeEnd('nodelist_update') + await nodeListCache.set(NODE_LIST_CACHE_KEY, Promise.resolve([...nodeList])); +} + +export async function getNodeList(page: number = 1, limit: number = 100): Promise { + const fullNodeList = await nodeListCache.getAndSet(NODE_LIST_CACHE_KEY, () => { + return Promise.resolve([...nodeList]); + }); + + const startIndex = (page - 1) * limit; + const endIndex = startIndex + limit; + const paginatedNodeList = fullNodeList.slice(startIndex, endIndex); + + return { + nodes: paginatedNodeList, + totalNodes: fullNodeList.length, + page: page, + limit: limit, + totalPages: Math.ceil(fullNodeList.length / limit) + }; +} + +export async function getNetworkAccount(): Promise { + return networkAccountCache.getAndSet(NETWORK_ACCOUNT_CACHE_KEY, async () => { + const response = await axios.get(`${getArchiverUrl().url}/get-network-account?hash=false`); + return response.data; + }); } export function removeFromNodeList(ip: string, port: string): void { From dd1fa802533c8e046c7e026ab0e17c853aa963e5 Mon Sep 17 00:00:00 2001 From: kenny-io Date: Fri, 18 Oct 2024 14:26:48 +0400 Subject: [PATCH 2/3] add cycleinfo method --- src/api.ts | 57 ++++++++++++++++++++++++++++++++++++++- src/external/Collector.ts | 31 +++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/api.ts b/src/api.ts index f6cc452e..6c4d7230 100755 --- a/src/api.ts +++ b/src/api.ts @@ -3784,7 +3784,6 @@ export const methods = { countFailedResponse(api_name, 'exception in /contract/accesslist') } }, - shardeum_getNodeList: async function (args: RequestParamsLike, callback: JSONRPCCallbackTypePlain) { const api_name = 'shardeum_getNodeList' const ticket = crypto @@ -3877,6 +3876,62 @@ export const methods = { countFailedResponse(api_name, 'Failed to fetch network account data'); } }, + shardeum_getCycleInfo: async function (args: RequestParamsLike, callback: JSONRPCCallbackTypePlain) { + const api_name = 'shardeum_getCycleInfo'; + nestedCountersInstance.countEvent('endpoint', api_name); + + if (!Array.isArray(args) || args.length > 1 || (args.length === 1 && typeof args[0] !== 'number' && args[0] !== null)) { + callback({ code: -32602, message: 'Invalid params: Expected a single number or null.' }, null); + countFailedResponse(api_name, 'Invalid params: Expected a single number or null.'); + return; + } + const ticket = crypto + .createHash('sha1') + .update(api_name + Math.random() + Date.now()) + .digest('hex'); + + logEventEmitter.emit('fn_start', ticket, api_name, performance.now()); + const cycleNumber = args.length === 1 ? args[0] : null; + try { + const result = await collectorAPI.getCycleInfo(cycleNumber); + + if (result) { + const cycleRecord = result.cycleRecord; + // transform the response to match the spec structure + const restructuredData = { + cycleCounter: cycleRecord.counter, + startTime: cycleRecord.start * 1000, + endTime: (cycleRecord.start + cycleRecord.duration) * 1000, + nodes: { + active: cycleRecord.active, + standby: cycleRecord.standby, + syncing: cycleRecord.syncing + }, + desired: cycleRecord.desired, + duration: cycleRecord.duration, + maxSyncTime: cycleRecord.maxSyncTime, + timestamp: cycleRecord.timestamp + }; + + const response = { + cycleInfo: restructuredData, + }; + + logEventEmitter.emit('fn_end', ticket, { success: true }, performance.now()); + callback(null, response); + countSuccessResponse(api_name, 'success', 'collector'); + } else { + logEventEmitter.emit('fn_end', ticket, { success: false }, performance.now()); + callback({ code: -32000, message: 'Cycle info not found' }, null); + countFailedResponse(api_name, 'Cycle info not found'); + } + } catch (error: any) { + console.error('Error fetching cycle info:', error); + logEventEmitter.emit('fn_end', ticket, { success: false }, performance.now()); + callback({ code: -32000, message: 'Failed to fetch cycle info', data: error.message }, null); + countFailedResponse(api_name, 'Failed to fetch cycle info'); + } + }, eth_subscribe: async function (args: RequestParamsLike, callback: JSONRPCCallbackTypePlain) { const api_name = 'eth_subscribe' diff --git a/src/external/Collector.ts b/src/external/Collector.ts index 6e09241f..23f9e753 100644 --- a/src/external/Collector.ts +++ b/src/external/Collector.ts @@ -563,6 +563,37 @@ class Collector extends BaseExternal { // } return result as readableTransaction } + async getCycleInfo(cycleNumber?: number): Promise { + if (!CONFIG.collectorSourcing.enabled) { + console.log('Collector sourcing is not enabled'); + return null; + } + nestedCountersInstance.countEvent('collector', 'getCycleInfo'); + const requestConfig: AxiosRequestConfig = { + method: 'get', + url: `${this.baseUrl}/api/cycleinfo`, + params: cycleNumber ? { cycleNumber: cycleNumber.toString() } : { count: '1' }, + headers: this.defaultHeaders, + }; + + try { + const res = await axiosWithRetry<{ success: boolean; cycles: any[] }>(requestConfig); + if (!res.data.success || !res.data.cycles || res.data.cycles.length === 0) { + console.log(`No cycles found in the response ${cycleNumber ? `for cycleNumber: ${cycleNumber}` : 'for latest cycle'}`); + return null; + } + const cycleInfo = res.data.cycles[0]; + return cycleInfo; + } catch (error) { + nestedCountersInstance.countEvent('collector', 'getCycleInfo-error'); + console.error('Collector: Error getting cycle info', error); + if (axios.isAxiosError(error)) { + console.error(`Request failed with status: ${error.response?.status}`); + console.error(`Error response data: ${JSON.stringify(error.response?.data)}`); + } + return null; + } + } } interface readableReceipt { From 4dfea37ad63fe61dff6b461dd2f8c2d6e883f191 Mon Sep 17 00:00:00 2001 From: kenny-io Date: Wed, 23 Oct 2024 21:30:23 +0400 Subject: [PATCH 3/3] update retry mechanism --- src/external/Archiver.ts | 1 - src/utils.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/external/Archiver.ts b/src/external/Archiver.ts index 37df3412..8fabf7e8 100644 --- a/src/external/Archiver.ts +++ b/src/external/Archiver.ts @@ -1,5 +1,4 @@ import { getNodeList, getNetworkAccount } from '../utils'; -import axios from 'axios'; export class Archiver { constructor() {} diff --git a/src/utils.ts b/src/utils.ts index a23fb0e8..a690a4c3 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -129,7 +129,7 @@ export async function updateNodeList(tryInfinate = false): Promise { console.log(`Updating NodeList from ${getArchiverUrl().url}`) console.time('nodelist_update') - const nRetry = tryInfinate ? -1 : 0 // infinitely retry or no retries + const nRetry = tryInfinate ? -1 : 5 // infinitely retry or 5 retries if initial request fails if (config.askLocalHostForArchiver === true) { if (gotArchiver === false) { gotArchiver = true @@ -202,7 +202,7 @@ export async function updateNodeList(tryInfinate = false): Promise { await nodeListCache.set(NODE_LIST_CACHE_KEY, Promise.resolve([...nodeList])); } -export async function getNodeList(page: number = 1, limit: number = 100): Promise { +export async function getNodeList(page: number, limit: number): Promise { const fullNodeList = await nodeListCache.getAndSet(NODE_LIST_CACHE_KEY, () => { return Promise.resolve([...nodeList]); });