From ac79a77af65232f58e5d2f260bed9fbbebd29faf Mon Sep 17 00:00:00 2001 From: Marc Fornos Date: Wed, 25 Oct 2023 17:50:42 +0200 Subject: [PATCH] add admin api --- package-lock.json | 123 ++++++++++++++++++++++++ package.json | 1 + src/server.ts | 4 + src/services/admin/routes.ts | 74 ++++++++++++++ src/services/auth.ts | 38 ++++++++ src/services/index.ts | 4 + src/services/monitoring/head-catcher.ts | 61 ++++++------ src/services/persistence/scheduler.ts | 9 ++ src/services/root.ts | 4 +- 9 files changed, 289 insertions(+), 29 deletions(-) create mode 100644 src/services/admin/routes.ts create mode 100644 src/services/auth.ts diff --git a/package-lock.json b/package-lock.json index dbdfd3f8..038606e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { + "@fastify/jwt": "^7.2.2", "@fastify/swagger": "^8.11.0", "@fastify/swagger-ui": "^1.10.0", "@sodazone/ocelloids": "^1.1.8", @@ -807,6 +808,18 @@ "fast-json-stringify": "^5.7.0" } }, + "node_modules/@fastify/jwt": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@fastify/jwt/-/jwt-7.2.2.tgz", + "integrity": "sha512-ZF0lyEjEIJnwqe0zjeSQkjfpAIrKdZfhTwUM+Z74NFEN+WodDi12cjABFPm2CrI8jtc4KInytSA74bN2jJ0MGQ==", + "dependencies": { + "@fastify/error": "^3.0.0", + "@lukeed/ms": "^2.0.0", + "fast-jwt": "^3.0.0", + "fastify-plugin": "^4.0.0", + "steed": "^1.1.3" + } + }, "node_modules/@fastify/send": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@fastify/send/-/send-2.1.0.tgz", @@ -3318,6 +3331,22 @@ "node": ">=8" } }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/asn1.js/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, "node_modules/async-mutex": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.4.0.tgz", @@ -4237,6 +4266,14 @@ "url": "https://github.com/motdotla/dotenv?sponsor=1" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.537", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.537.tgz", @@ -4786,6 +4823,20 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, + "node_modules/fast-jwt": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-3.3.1.tgz", + "integrity": "sha512-1YuuIJeh1hEvfcYDe89P2oGACWI5hd2GadRDKHalSxkc1Z0z8I6yzuVK6SF15sW09QZngTV6d7g4+TFL9bvs5A==", + "dependencies": { + "@lukeed/ms": "^2.0.1", + "asn1.js": "^5.4.1", + "ecdsa-sig-formatter": "^1.0.11", + "mnemonist": "^0.39.5" + }, + "engines": { + "node": ">=16 <22" + } + }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", @@ -4819,6 +4870,17 @@ "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-2.2.0.tgz", "integrity": "sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==" }, + "node_modules/fastfall": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/fastfall/-/fastfall-1.5.1.tgz", + "integrity": "sha512-KH6p+Z8AKPXnmA7+Iz2Lh8ARCMr+8WNPVludm1LGkZoD2MjY6LVnRMtTKhkdzI+jr0RzQWXKzKyBJm1zoHEL4Q==", + "dependencies": { + "reusify": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fastify": { "version": "4.23.2", "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.23.2.tgz", @@ -4888,6 +4950,15 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "node_modules/fastparallel": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/fastparallel/-/fastparallel-2.4.1.tgz", + "integrity": "sha512-qUmhxPgNHmvRjZKBFUNI0oZuuH9OlSIOXmJ98lhKPxMZZ7zS/Fi0wRHOihDSz0R1YiIOjxzOY4bq65YTcdBi2Q==", + "dependencies": { + "reusify": "^1.0.4", + "xtend": "^4.0.2" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -4896,6 +4967,15 @@ "reusify": "^1.0.4" } }, + "node_modules/fastseries": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/fastseries/-/fastseries-1.7.2.tgz", + "integrity": "sha512-dTPFrPGS8SNSzAt7u/CbMKCJ3s01N04s4JFbORHcmyvVfVKmbhMD1VtRbh5enGHxkaQDqWyLefiKOGGmohGDDQ==", + "dependencies": { + "reusify": "^1.0.0", + "xtend": "^4.0.0" + } + }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", @@ -7698,6 +7778,11 @@ "resolved": "https://registry.npmjs.org/mingo/-/mingo-6.4.7.tgz", "integrity": "sha512-9CxaYNHl64KG/IPhIpTzNHajHB7HMj51bKy51Gdi3QzLOEbpK2mWzvs0U/3GCysytBycrkTjbNSP6GZ0rgh0hg==" }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -7719,6 +7804,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mnemonist": { + "version": "0.39.5", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.5.tgz", + "integrity": "sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ==", + "dependencies": { + "obliterator": "^2.0.1" + } + }, "node_modules/mock-socket": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/mock-socket/-/mock-socket-9.3.1.tgz", @@ -7949,6 +8042,11 @@ "node": ">=8" } }, + "node_modules/obliterator": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.4.tgz", + "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ==" + }, "node_modules/on-exit-leak-free": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz", @@ -8823,6 +8921,11 @@ "node": ">=10" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "node_modules/secure-json-parse": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", @@ -9014,6 +9117,18 @@ "node": ">= 0.8" } }, + "node_modules/steed": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/steed/-/steed-1.1.3.tgz", + "integrity": "sha512-EUkci0FAUiE4IvGTSKcDJIQ/eRUP2JJb56+fvZ4sdnguLTqIdKjSxUe138poW8mkvKWXW2sFPrgTsxqoISnmoA==", + "dependencies": { + "fastfall": "^1.5.0", + "fastparallel": "^2.2.0", + "fastq": "^1.3.0", + "fastseries": "^1.7.0", + "reusify": "^1.0.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -9613,6 +9728,14 @@ } } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 0673aafd..4f82ca80 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "typescript": "^5.2.2" }, "dependencies": { + "@fastify/jwt": "^7.2.2", "@fastify/swagger": "^8.11.0", "@fastify/swagger-ui": "^1.10.0", "@sodazone/ocelloids": "^1.1.8", diff --git a/src/server.ts b/src/server.ts index bd8ef9c4..328a37fd 100644 --- a/src/server.ts +++ b/src/server.ts @@ -13,6 +13,8 @@ import { logger } from './environment.js'; import { ServerOptions } from './types.js'; import { Root, + Auth, + Administration, Persistence, Configuration, Monitoring, @@ -71,10 +73,12 @@ export async function createServer( }); await server.register(Root); + await server.register(Auth); await server.register(Configuration, opts); await server.register(Persistence, opts); await server.register(Connector); await server.register(Monitoring); + await server.register(Administration); return server; } diff --git a/src/services/admin/routes.ts b/src/services/admin/routes.ts new file mode 100644 index 00000000..8aff803b --- /dev/null +++ b/src/services/admin/routes.ts @@ -0,0 +1,74 @@ +import { FastifyInstance } from 'fastify'; + +export default async function Administration( + fastify: FastifyInstance +) { + const { log, storage: { root }, scheduler } = fastify; + + fastify.delete( + '/admin/storage/root', + { + onRequest: [fastify.auth] + }, + async (request, reply) => { + log.warn( + 'Clearing root database %s %j', + request.ip, + request.headers + ); + await root.clear(); + reply.send(); + } + ); + + fastify.get('/admin/scheduled',{ + onRequest: [fastify.auth], + schema: { + hide: true, + response: { + 200: { + type: 'array', + items: { + type: 'string' + } + } + } + } + }, async (_, reply) => { + reply.send(await scheduler.allTaskTimes()); + }); + + fastify.get<{ + Params: { + id: string + } + }>('/admin/scheduled/:id',{ + onRequest: [fastify.auth], + schema: { + hide: true + } + }, async (request, reply) => { + reply.send( + await scheduler.getById(request.params.id) + ); + }); + + fastify.delete<{ + Params: { + id: string + } + }>('/admin/scheduled/:id',{ + onRequest: [fastify.auth], + schema: { + hide: true, + response: { + 200: { + type: 'null' + } + } + } + }, async (request, reply) => { + await scheduler.remove(request.params.id); + reply.send(); + }); +} \ No newline at end of file diff --git a/src/services/auth.ts b/src/services/auth.ts new file mode 100644 index 00000000..73f58429 --- /dev/null +++ b/src/services/auth.ts @@ -0,0 +1,38 @@ +import { FastifyPluginAsync, FastifyReply, FastifyRequest } from 'fastify'; +import fp from 'fastify-plugin'; +import jwt from '@fastify/jwt'; + +import { environment } from '../environment.js'; + +declare module 'fastify' { + interface FastifyInstance { + auth: ( + request: FastifyRequest, reply: FastifyReply + ) => Promise + } +} + +const authPlugin: FastifyPluginAsync += async fastify => { + if (environment !== 'development' && !process.env.XCMON_SECRET) { + fastify.log.warn('!! Default XCMON_SECRET configured !!'); + } + + fastify.register(jwt, { + secret: process.env.XCMON_SECRET || 'IAOAbraxasSabaoth' + }); + fastify.decorate('auth', + async function ( + request: FastifyRequest, reply: FastifyReply + ) : Promise { + try { + await request.jwtVerify(); + } catch (err) { + reply.status(401).send(err); + } + } + ); +}; + +export default fp(authPlugin); + diff --git a/src/services/index.ts b/src/services/index.ts index 557d94ff..c8f38901 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1,4 +1,6 @@ import Root from './root.js'; +import Auth from './auth.js'; +import Administration from './admin/routes.js'; import Configuration from './configuration.js'; import Monitoring from './monitoring/plugin.js'; import Persistence from './persistence/plugin.js'; @@ -8,6 +10,8 @@ export * from './types.js'; export { Root, + Auth, + Administration, Persistence, Configuration, Monitoring, diff --git a/src/services/monitoring/head-catcher.ts b/src/services/monitoring/head-catcher.ts index dcb3e162..5a2ff93e 100644 --- a/src/services/monitoring/head-catcher.ts +++ b/src/services/monitoring/head-catcher.ts @@ -81,14 +81,21 @@ export class HeadCatcher extends EventEmitter { const blockPipe = api.pipe( blocks(), - tap(b => { + retryWithTruncatedExpBackoff(), + tap(async ({ block: {header} }) => { this.#log.info( '[%s] SEEN block #%s %s', chainId, - b.block.header.number.toString(), - b.block.header.hash.toHex() - );}), - retryWithTruncatedExpBackoff() + header.number.toString(), + header.hash.toHex() + ); + // TODO: revisit janitor tasks in side effects + const blockHash = header.hash.toHex(); + await this.#janitor.schedule({ + sublevel: chainId + ':blocks', + key: blockHash + }); + }) ); const paraPipe = blockPipe.pipe( mergeMap(block => { @@ -111,6 +118,19 @@ export class HeadCatcher extends EventEmitter { hrmpMessages, umpMessages }; + }), + // TODO: see other taps + tap(async ({ block: { block: { header }} }) => { + // TODO: revisit janitor tasks in side effects + const blockHash = header.hash.toHex(); + await this.#janitor.schedule({ + sublevel: chainId + ':blocks', + key: 'hrmp-messages:' + blockHash + }, + { + sublevel: chainId + ':blocks', + key: 'ump-messages:' + blockHash + }); }) ); }) @@ -197,9 +217,14 @@ export class HeadCatcher extends EventEmitter { mergeMap(head => from(this.#getBlock( chainId, api, head.hash.toHex() ))), - // retryWithTruncatedExpBackoff(), + retryWithTruncatedExpBackoff(), // Revisit: clean up as a side effect? - tap(this.#updateJanitorTasks(chainId)), + tap(async ({ block: { header } }) => { + const blockHash = header.hash.toHex(); + await this.#blockCache(chainId).del(blockHash); + // TODO: clean up storage related to the block? + // will require additional indexing + }), share() ); } else { @@ -350,7 +375,7 @@ export class HeadCatcher extends EventEmitter { mergeMap(head => defer( () => this.#doCatchUp(chainId, api, head) )), - // retryWithTruncatedExpBackoff(), + retryWithTruncatedExpBackoff(), mergeMap(head => head) ); }; @@ -441,24 +466,4 @@ export class HeadCatcher extends EventEmitter { author: block.author?.toU8a() })); } - - #updateJanitorTasks(chainId: string) { - return ({ block: { header } }: SignedBlockExtended) => { - const blockHash = header.hash.toHex(); - this.#janitor.schedule( - { - sublevel: chainId + ':blocks', - key: 'hrmp-messages:' + blockHash - }, - { - sublevel: chainId + ':blocks', - key: 'ump-messages:' + blockHash - }, - { - sublevel: chainId + ':blocks', - key: blockHash - } - ); - }; - } } \ No newline at end of file diff --git a/src/services/persistence/scheduler.ts b/src/services/persistence/scheduler.ts index 2975f7c1..c0e4fc1c 100644 --- a/src/services/persistence/scheduler.ts +++ b/src/services/persistence/scheduler.ts @@ -1,6 +1,7 @@ import Stream from 'node:stream'; import { DB, Family, Logger } from '../types.js'; +import { NotFound } from '../../errors.js'; export type Scheduled = { key?: string @@ -78,6 +79,14 @@ export class Scheduler extends Stream.EventEmitter { return await this.#tasks.keys().all(); } + async getById(key: string) { + try { + return await this.#tasks.get(key); + } catch (error) { + throw new NotFound('Task no found'); + } + } + async #run() { const cancellable = new Promise(resolve => { this.#cancel = resolve; diff --git a/src/services/root.ts b/src/services/root.ts index 220a9bb5..9e5c9c37 100644 --- a/src/services/root.ts +++ b/src/services/root.ts @@ -2,7 +2,9 @@ import { FastifyInstance } from 'fastify'; import version from '../version.js'; -export default async function Root(fastify: FastifyInstance) { +export default async function Root( + fastify: FastifyInstance +) { fastify.get('/',{ schema: { hide: true,