diff --git a/jest.config.js b/jest.config.js index 26c527a5e7..7b32c63a0a 100644 --- a/jest.config.js +++ b/jest.config.js @@ -62,7 +62,7 @@ module.exports = { // forceCoverageMatch: [], // A path to a module which exports an async function that is triggered once before all test suites - // globalSetup: undefined, + // globalSetup: undefined // A path to a module which exports an async function that is triggered once after all test suites // globalTeardown: undefined, @@ -94,6 +94,7 @@ module.exports = { '^@subql/common/(.*)$': '/packages/common/src/$1', '^@subql/node-core/(.*)$': '/packages/node-core/src/$1', '^@subql/utils/(.*)$': '/packages/utils/src/$1', + '^@subql/node-core/logger': '/packages/node-core/logger', }, // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader @@ -140,7 +141,7 @@ module.exports = { // The paths to modules that run some code to configure or set up the testing environment before each test // setupFiles: [], - setupFiles: ['./test/jest-setup.ts'], + setupFiles: ['/test/jest-setup.ts'], // A list of paths to modules that run some code to configure or set up the testing framework before each test // setupFilesAfterEnv: [], diff --git a/packages/cli/package.json b/packages/cli/package.json index b3267eef3e..e620f49b7b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -17,7 +17,7 @@ "@subql/common-avalanche": "latest", "@subql/common-cosmos": "latest", "@subql/common-substrate": "workspace:*", - "@subql/common-terra": "^0.6.0", + "@subql/common-terra": "latest", "@subql/utils": "workspace:*", "@subql/validator": "workspace:*", "@types/ejs": "^3.1.0", diff --git a/packages/common/fixtures/package.json b/packages/common/fixtures/package.json index 60b16ae168..f23f0ec828 100644 --- a/packages/common/fixtures/package.json +++ b/packages/common/fixtures/package.json @@ -9,7 +9,6 @@ "test": "jest", "codegen": "./node_modules/.bin/subql codegen" }, - "homepage": "https://github.com/subquery/subql-starter", "repository": "github:subquery/subql-starter", "files": [ @@ -21,7 +20,7 @@ "license": "Apache-2.0", "devDependencies": { "@polkadot/api": "^3", - "@subql/types": "^0.6.0", + "@subql/types": "latest", "typescript": "^4.1.3", "@subql/cli": "^0.7.3" } diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index 20455f8bcf..27be569151 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 All logs must start with the format: [x.y.z] - yyyy-mm-dd ## [Unreleased] +### Changes +- Moved `yargs` file from `node-core` to `node`. (#1281) ## [0.1.1] - 2022-08-26 ### Fixed diff --git a/packages/node-core/logger.js b/packages/node-core/logger.js new file mode 100644 index 0000000000..9ca37e344a --- /dev/null +++ b/packages/node-core/logger.js @@ -0,0 +1,7 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +// A standalone entrypoint for logger so we can import `@subql/node-core/logger` +const logger = require('./dist/logger'); + +module.exports = logger; diff --git a/packages/node-core/package.json b/packages/node-core/package.json index dc4eb6d454..2fe82bbef3 100644 --- a/packages/node-core/package.json +++ b/packages/node-core/package.json @@ -1,6 +1,6 @@ { "name": "@subql/node-core", - "version": "0.1.2-1", + "version": "0.1.2-2", "description": "Common node features that are agnostic to blockchains", "homepage": "https://github.com/subquery/subql", "repository": "github:subquery/subql", @@ -14,7 +14,8 @@ "license": "Apache-2.0", "files": [ "src/global.d.ts", - "/dist" + "/dist", + "logger.js" ], "dependencies": { "@nestjs/common": "^8.2.6", diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts index b55458867d..dc721d7615 100644 --- a/packages/node-core/src/configure/NodeConfig.ts +++ b/packages/node-core/src/configure/NodeConfig.ts @@ -33,6 +33,12 @@ export interface IConfig { readonly ipfs?: string; readonly dictionaryTimeout: number; readonly workers?: number; + readonly profiler?: boolean; + readonly migrate: boolean; + readonly unsafe?: boolean; + readonly subscription: boolean; + readonly disableHistorical: boolean; + readonly reindex?: number; } export type MinConfig = Partial> & Pick; @@ -48,6 +54,10 @@ const DEFAULT_CONFIG = { timestampField: true, proofOfIndex: false, dictionaryTimeout: 30, + profiler: false, + migrate: false, + subscription: false, + disableHistorical: true, }; export class NodeConfig implements IConfig { @@ -154,6 +164,26 @@ export class NodeConfig implements IConfig { return this._config.workers; } + get profiler(): boolean { + return this._config.profiler; + } + + get migrate(): boolean { + return this._config.migrate; + } + + get unsafe(): boolean { + return this._config.unsafe; + } + + get subscription(): boolean { + return this._config.subscription; + } + + get disableHistorical(): boolean { + return this._config.disableHistorical; + } + merge(config: Partial): this { assign(this._config, config); return this; diff --git a/packages/node-core/src/db/db.module.test.ts b/packages/node-core/src/db/db.module.test.ts index b155369422..9e7b393774 100644 --- a/packages/node-core/src/db/db.module.test.ts +++ b/packages/node-core/src/db/db.module.test.ts @@ -4,9 +4,12 @@ import {INestApplication} from '@nestjs/common'; import {Test} from '@nestjs/testing'; import {Sequelize} from 'sequelize'; +import {NodeConfig} from '../configure/NodeConfig'; import {SubqueryRepo} from '../entities'; import {DbModule} from './db.module'; +const nodeConfig = new NodeConfig({subquery: 'packages/node-core/test/v1.0.0', subqueryName: 'test'}); + describe('DbModule', () => { let app: INestApplication; @@ -16,15 +19,7 @@ describe('DbModule', () => { it('can connect to database', async () => { const module = await Test.createTestingModule({ - imports: [ - DbModule.forRoot({ - host: process.env.DB_HOST ?? '127.0.0.1', - port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, - username: process.env.DB_USER ?? 'postgres', - password: process.env.DB_PASS ?? 'postgres', - database: process.env.DB_DATABASE ?? 'postgres', - }), - ], + imports: [DbModule.forRootWithConfig(nodeConfig)], }).compile(); app = module.createNestApplication(); @@ -35,16 +30,7 @@ describe('DbModule', () => { it('can load subquery model', async () => { const module = await Test.createTestingModule({ - imports: [ - DbModule.forRoot({ - host: process.env.DB_HOST ?? '127.0.0.1', - port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, - username: process.env.DB_USER ?? 'postgres', - password: process.env.DB_PASS ?? 'postgres', - database: process.env.DB_DATABASE ?? 'postgres', - }), - DbModule.forFeature(['Subquery']), - ], + imports: [DbModule.forRootWithConfig(nodeConfig), DbModule.forFeature(['Subquery'])], }).compile(); app = module.createNestApplication(); diff --git a/packages/node-core/src/db/db.module.ts b/packages/node-core/src/db/db.module.ts index 015fcda344..133bcffb10 100644 --- a/packages/node-core/src/db/db.module.ts +++ b/packages/node-core/src/db/db.module.ts @@ -3,10 +3,10 @@ import {DynamicModule, Global} from '@nestjs/common'; import {Sequelize, Options as SequelizeOption} from 'sequelize'; +import {NodeConfig} from '../configure/NodeConfig'; import * as entities from '../entities'; import {getLogger} from '../logger'; import {delay} from '../utils/promise'; -import {getYargsOption} from '../yargs'; export interface DbOption { host: string; @@ -18,6 +18,14 @@ export interface DbOption { const logger = getLogger('db'); +const DEFAULT_DB_OPTION: DbOption = { + host: process.env.DB_HOST ?? '127.0.0.1', + port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, + username: process.env.DB_USER ?? 'postgres', + password: process.env.DB_PASS ?? 'postgres', + database: process.env.DB_DATABASE ?? 'postgres', +}; + async function establishConnection(sequelize: Sequelize, numRetries: number): Promise { try { await sequelize.authenticate(); @@ -32,37 +40,68 @@ async function establishConnection(sequelize: Sequelize, numRetries: number): Pr } } -const sequelizeFactory = (option: SequelizeOption) => async () => { +const sequelizeFactory = (option: SequelizeOption, migrate: boolean) => async () => { const sequelize = new Sequelize(option); const numRetries = 5; await establishConnection(sequelize, numRetries); for (const factoryFn of Object.keys(entities).filter((k) => /Factory$/.exec(k))) { entities[factoryFn as keyof typeof entities](sequelize); } - const {migrate} = getYargsOption().argv; await sequelize.sync({alter: migrate}); return sequelize; }; @Global() export class DbModule { - static forRoot(option: DbOption): DynamicModule { - const {argv} = getYargsOption(); + static forRootWithConfig(nodeConfig: NodeConfig, option: DbOption = DEFAULT_DB_OPTION): DynamicModule { + const logger = getLogger('db'); + + const factory = sequelizeFactory( + { + ...option, + dialect: 'postgres', + logging: nodeConfig.debug + ? (sql: string, timing?: number) => { + logger.debug(sql); + } + : false, + }, + nodeConfig.migrate + )(); + + return { + module: DbModule, + providers: [ + { + provide: Sequelize, + useFactory: () => factory, + }, + ], + exports: [Sequelize], + }; + } + + static forRoot(option: DbOption = DEFAULT_DB_OPTION): DynamicModule { const logger = getLogger('db'); return { module: DbModule, providers: [ { provide: Sequelize, - useFactory: sequelizeFactory({ - ...option, - dialect: 'postgres', - logging: argv.debug - ? (sql: string, timing?: number) => { - logger.debug(sql); - } - : false, - }), + useFactory: (nodeConfig: NodeConfig) => + sequelizeFactory( + { + ...option, + dialect: 'postgres', + logging: nodeConfig.debug + ? (sql: string, timing?: number) => { + logger.debug(sql); + } + : false, + }, + nodeConfig.migrate + )(), + inject: [NodeConfig], }, ], exports: [Sequelize], diff --git a/packages/node-core/src/index.ts b/packages/node-core/src/index.ts index dd14fd09da..1f0236f6e5 100644 --- a/packages/node-core/src/index.ts +++ b/packages/node-core/src/index.ts @@ -3,7 +3,6 @@ export * from './api.service'; export * from './logger'; -export * from './yargs'; export * from './profiler'; export * from './events'; export * from './configure'; diff --git a/packages/node-core/src/indexer/benchmark.service.ts b/packages/node-core/src/indexer/benchmark.service.ts index 7e9637950a..b196087106 100644 --- a/packages/node-core/src/indexer/benchmark.service.ts +++ b/packages/node-core/src/indexer/benchmark.service.ts @@ -5,14 +5,13 @@ import {OnEvent} from '@nestjs/event-emitter'; import {Interval} from '@nestjs/schedule'; import dayjs from 'dayjs'; import duration from 'dayjs/plugin/duration'; +import {NodeConfig} from '../configure'; import {IndexerEvent, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload} from '../events'; import {getLogger} from '../logger'; import {delay} from '../utils/promise'; -import {getYargsOption} from '../yargs'; const SAMPLING_TIME_VARIANCE = 15; const logger = getLogger('benchmark'); -const {argv} = getYargsOption(); dayjs.extend(duration); export class BenchmarkService { @@ -26,6 +25,8 @@ export class BenchmarkService { private currentProcessedBlockAmount: number; private lastProcessedBlockAmount: number; + constructor(private nodeConfig: NodeConfig) {} + @Interval(SAMPLING_TIME_VARIANCE * 1000) async benchmark(): Promise { if (!this.currentProcessingHeight || !this.currentProcessingTimestamp || !this.currentProcessedBlockAmount) { @@ -44,7 +45,7 @@ export class BenchmarkService { const days = Math.floor(blockDuration.asDays()); const durationStr = `${days} days ${hoursMinsStr}`; - if (argv.profiler) { + if (this.nodeConfig.profiler) { logger.info( `Processed ${ this.currentProcessedBlockAmount - this.lastProcessedBlockAmount diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 940fbbb0e9..4e723d8b9f 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -45,14 +45,12 @@ import { camelCaseObjectKey, makeTriggerName, } from '../utils'; -import {getYargsOption} from '../yargs'; import {Metadata, MetadataFactory, MetadataRepo, PoiFactory, PoiRepo, ProofOfIndex} from './entities'; import {StoreOperations} from './StoreOperations'; import {OperationType} from './types'; const logger = getLogger('store'); const NULL_MERKEL_ROOT = hexToU8a('0x00'); -const {argv} = getYargsOption(); const NotifyTriggerManipulationType = [`INSERT`, `DELETE`, `UPDATE`]; const KEY_FIELDS = ['id', '__id', '__block_range']; @@ -157,7 +155,7 @@ export class StoreService { enumTypeMap.set(e.name, `"${enumTypeName}"`); } const extraQueries = []; - if (argv.subscription) { + if (this.config.subscription) { extraQueries.push(createSendNotificationTriggerFunction); } for (const model of this.modelsRelations.models) { @@ -187,7 +185,7 @@ export class StoreService { this.addScopeAndBlockHeightHooks(sequelizeModel); extraQueries.push(createExcludeConstraintQuery(schema, sequelizeModel.tableName)); } - if (argv.subscription) { + if (this.config.subscription) { const triggerName = makeTriggerName(schema, sequelizeModel.tableName); const triggers = await this.sequelize.query(getNotifyTriggers(), { replacements: {triggerName}, @@ -282,7 +280,7 @@ export class StoreService { enabled = false; } } catch (e) { - enabled = !argv['disable-historical']; + enabled = !this.config.disableHistorical; } logger.info(`Historical state is ${enabled ? 'enabled' : 'disabled'}`); return enabled; diff --git a/packages/node-core/src/logger.ts b/packages/node-core/src/logger.ts index efa9d82463..ec7b12eb91 100644 --- a/packages/node-core/src/logger.ts +++ b/packages/node-core/src/logger.ts @@ -1,21 +1,25 @@ // Copyright 2020-2022 OnFinality Limited authors & contributors // SPDX-License-Identifier: Apache-2.0 -import { isMainThread, threadId } from 'node:worker_threads'; -import { LoggerService } from '@nestjs/common'; -import { Logger } from '@subql/utils'; +import {isMainThread, threadId} from 'node:worker_threads'; +import {LoggerService} from '@nestjs/common'; +import {Logger} from '@subql/utils'; import Pino from 'pino'; -import { argv } from './yargs'; -const outputFmt = argv('output-fmt') as 'json' | 'colored'; -const debug = argv('debug'); -const logLevel = argv('log-level') as string | undefined; +let logger: Logger; -const logger = new Logger({ - level: debug ? 'debug' : logLevel, - outputFormat: outputFmt, - nestedKey: 'payload', -}); +export function initLogger(debug = false, outputFmt?: 'json' | 'colored', logLevel?: string): void { + logger = new Logger({ + level: debug ? 'debug' : logLevel, + outputFormat: outputFmt, + nestedKey: 'payload', + }); +} + +// Init logger for tests +if ((global as any).__TEST__) { + initLogger(); +} export function getLogger(category: string): Pino.Logger { return logger.getLogger(category); @@ -26,13 +30,11 @@ export function setLevel(level: Pino.LevelWithSilent): void { } export class NestLogger implements LoggerService { - private logger = logger.getLogger( - `nestjs${isMainThread ? '-0' : `-#${threadId}`}`, - ); + private logger = logger.getLogger(`nestjs${isMainThread ? '-0' : `-#${threadId}`}`); error(message: any, trace?: string): void { if (trace) { - this.logger.error({ trace }, message); + this.logger.error({trace}, message); } else { this.logger.error(message); } diff --git a/packages/node-core/src/profiler.ts b/packages/node-core/src/profiler.ts index 7decba929d..34b0ce7f10 100644 --- a/packages/node-core/src/profiler.ts +++ b/packages/node-core/src/profiler.ts @@ -3,19 +3,15 @@ /* class decorator */ -import { getLogger } from './logger'; +import {getLogger} from './logger'; function isPromise(e: any): boolean { return !!e && typeof e.then === 'function'; } + const logger = getLogger('profiler'); -function printCost( - start: Date, - end: Date, - target: string, - method: string, -): void { +function printCost(start: Date, end: Date, target: string, method: string): void { logger.info(`${target}, ${method}, ${end.getTime() - start.getTime()} ms`); } export function profiler(enabled = true): any { @@ -35,7 +31,7 @@ export function profiler(enabled = true): any { (err: any) => { printCost(start, new Date(), target.constructor.name, name); throw err; - }, + } ); } else { printCost(start, new Date(), target.constructor.name, name); @@ -62,7 +58,7 @@ export const profilerWrap = (err: any) => { printCost(start, new Date(), target, name); throw err; - }, + } ); } else { printCost(start, new Date(), target, name); diff --git a/packages/node-core/src/utils/batch-size.ts b/packages/node-core/src/utils/batch-size.ts index 01fb2600e2..c576afb8a0 100644 --- a/packages/node-core/src/utils/batch-size.ts +++ b/packages/node-core/src/utils/batch-size.ts @@ -2,22 +2,22 @@ // SPDX-License-Identifier: Apache-2.0 import {getHeapStatistics} from 'v8'; +import {NodeConfig} from '../configure/NodeConfig'; import {getLogger} from '../logger'; -import {getYargsOption} from '../yargs'; const HIGH_THRESHOLD = 0.85; const LOW_THRESHOLD = 0.6; -const {argv} = getYargsOption(); const logger = getLogger('memory'); -export function checkMemoryUsage(batchSizeScale: number): number { +export function checkMemoryUsage(batchSizeScale: number, nodeConfig: NodeConfig): number { const memoryData = getHeapStatistics(); const ratio = memoryData.used_heap_size / memoryData.heap_size_limit; - if (argv.profiler) { + if (nodeConfig.profiler) { logger.info(`Heap Statistics: ${JSON.stringify(memoryData)}`); logger.info(`Heap Usage: ${ratio}`); } + let scale = batchSizeScale; if (ratio > HIGH_THRESHOLD) { diff --git a/packages/node-core/src/utils/index.ts b/packages/node-core/src/utils/index.ts index fb16c4f0dd..93418b9aa8 100644 --- a/packages/node-core/src/utils/index.ts +++ b/packages/node-core/src/utils/index.ts @@ -7,3 +7,4 @@ export * from './graphql'; export * from './sync-helper'; export * from './batch-size'; export * from './autoQueue'; +export * from './project'; diff --git a/packages/node-core/src/utils/project.ts b/packages/node-core/src/utils/project.ts new file mode 100644 index 0000000000..cade900301 --- /dev/null +++ b/packages/node-core/src/utils/project.ts @@ -0,0 +1,49 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import {QueryTypes, Sequelize} from 'sequelize'; +import {NodeConfig} from '../configure/NodeConfig'; +import {SubqueryRepo} from '../entities/Subquery.entity'; +import {MetadataRepo} from '../indexer/entities/Metadata.entity'; +import {getLogger} from '../logger'; + +const logger = getLogger('Project-Utils'); + +export async function getExistingProjectSchema( + nodeConfig: NodeConfig, + sequelize: Sequelize, + subqueryRepo: SubqueryRepo +): Promise { + const DEFAULT_DB_SCHEMA = 'public'; + let schema = nodeConfig.localMode ? DEFAULT_DB_SCHEMA : nodeConfig.dbSchema; + + let schemas: string[]; + try { + const result = await sequelize.query(`SELECT schema_name FROM information_schema.schemata`, { + type: QueryTypes.SELECT, + }); + schemas = result.map((x: any) => x.schema_name) as [string]; + } catch (err) { + logger.error(`Unable to fetch all schemas: ${err}`); + process.exit(1); + } + if (!schemas.includes(schema)) { + // fallback to subqueries table + const subqueryModel = await subqueryRepo.findOne({ + where: {name: nodeConfig.subqueryName}, + }); + if (subqueryModel) { + schema = subqueryModel.dbSchema; + } else { + schema = undefined; + } + } + return schema; +} + +export async function getMetaDataInfo(metadataRepo: MetadataRepo, key: string): Promise { + const res = await metadataRepo.findOne({ + where: {key: key}, + }); + return res?.value as number | undefined; +} diff --git a/packages/node-core/test/v1.0.0/distMock/chaintypes.js b/packages/node-core/test/v1.0.0/distMock/chaintypes.js new file mode 100644 index 0000000000..63642b3e18 --- /dev/null +++ b/packages/node-core/test/v1.0.0/distMock/chaintypes.js @@ -0,0 +1,3 @@ +'use strict'; + +exports.default = {typesBundle: {}}; diff --git a/packages/node-core/test/v1.0.0/distMock/index.js b/packages/node-core/test/v1.0.0/distMock/index.js new file mode 100644 index 0000000000..951d788ee5 --- /dev/null +++ b/packages/node-core/test/v1.0.0/distMock/index.js @@ -0,0 +1,6 @@ +'use strict'; +Object.defineProperty(exports, '__esModule', {value: true}); +const tslib_1 = require('tslib'); +//Exports all handler functions +require('@polkadot/api-augment'); +(0, tslib_1.__exportStar)(require('./mappings/mappingHandlers'), exports); diff --git a/packages/node-core/test/v1.0.0/erc20.abi.json b/packages/node-core/test/v1.0.0/erc20.abi.json new file mode 100644 index 0000000000..405d6b3648 --- /dev/null +++ b/packages/node-core/test/v1.0.0/erc20.abi.json @@ -0,0 +1,222 @@ +[ + { + "constant": true, + "inputs": [], + "name": "name", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "approve", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_from", + "type": "address" + }, + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transferFrom", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "decimals", + "outputs": [ + { + "name": "", + "type": "uint8" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + } + ], + "name": "balanceOf", + "outputs": [ + { + "name": "balance", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "symbol", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transfer", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + }, + { + "name": "_spender", + "type": "address" + } + ], + "name": "allowance", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "payable": true, + "stateMutability": "payable", + "type": "fallback" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "owner", + "type": "address" + }, + { + "indexed": true, + "name": "spender", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Approval", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "from", + "type": "address" + }, + { + "indexed": true, + "name": "to", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + } +] diff --git a/packages/node-core/test/v1.0.0/package.json b/packages/node-core/test/v1.0.0/package.json new file mode 100644 index 0000000000..b35bd0cf3a --- /dev/null +++ b/packages/node-core/test/v1.0.0/package.json @@ -0,0 +1,30 @@ +{ + "name": "subquery-starter", + "version": "0.0.4", + "description": "This project can be use as a starting point for developing your SubQuery project", + "main": "dist/index.js", + "scripts": { + "build": "tsc -b", + "prepack": "rm -rf dist && npm build", + "test": "jest", + "codegen": "./node_modules/.bin/subql codegen" + }, + "homepage": "https://github.com/subquery/subql-starter", + "repository": "github:subquery/subql-starter", + "files": [ + "dist", + "schema.graphql", + "project.yaml" + ], + "author": "Ian He & Jay Ji", + "license": "Apache-2.0", + "devDependencies": { + "@polkadot/api": "^8", + "@subql/types": "latest", + "typescript": "^4.1.3", + "@subql/cli": "latest" + }, + "resolutions": { + "ipfs-unixfs": "6.0.6" + } +} diff --git a/packages/node-core/test/v1.0.0/project.yaml b/packages/node-core/test/v1.0.0/project.yaml new file mode 100644 index 0000000000..cc51b347bc --- /dev/null +++ b/packages/node-core/test/v1.0.0/project.yaml @@ -0,0 +1,57 @@ +specVersion: 1.0.0 +name: moonriver +version: 1.0.0 +description: '' +repository: '' +runner: + node: + name: '@subql/node' + version: '>=1.0.0' + query: + name: '@subql/query' + version: '*' +schema: + file: ./schema.graphql +network: + chainId: '0x401a1f9dca3da46f5c4091016c8a2f26dcea05865116b286f60f668207d1474b' + endpoint: wss://moonriver.api.onfinality.io/public-ws + dictionary: 'https://api.subquery.network/sq/subquery/moonriver-dictionary' + chaintypes: + file: './distMock/chaintypes.js' +dataSources: + - kind: substrate/Runtime + startBlock: 69219 + mapping: + file: './distMock/index.js' + handlers: + - handler: collatorJoined + kind: substrate/EventHandler + filter: + module: parachainStaking + method: JoinedCollatorCandidates + - handler: collatorLeft + kind: substrate/CallHandler + filter: + module: parachainStaking + method: leaveCandidates + + - kind: substrate/Moonbeam + startBlock: 562683 + processor: + file: './dist/moonbeam.js' + # file: './node_modules/@subql/contract-processors/dist/moonbeam.js' + options: + # Must be a key of assets + abi: erc20 + address: '0x6bd193ee6d2104f14f94e2ca6efefae561a4334b' + assets: + erc20: + file: './erc20.abi.json' + mapping: + file: ./distMock/index.js + handlers: + - handler: erc20Transfer + kind: substrate/MoonbeamEvent + filter: + topics: + - Transfer(address indexed from,address indexed to,uint256 value) diff --git a/packages/node-core/test/v1.0.0/schema.graphql b/packages/node-core/test/v1.0.0/schema.graphql new file mode 100644 index 0000000000..4ef53ed683 --- /dev/null +++ b/packages/node-core/test/v1.0.0/schema.graphql @@ -0,0 +1,11 @@ +type StarterEntity @entity { + id: ID! #id is a required field + field1: Int! + + field2: String #filed2 is an optional field + field3: BigInt + + field4: Date + + field5: Boolean +} diff --git a/packages/node-core/test/v1.0.0/types.yaml b/packages/node-core/test/v1.0.0/types.yaml new file mode 100644 index 0000000000..4346d24204 --- /dev/null +++ b/packages/node-core/test/v1.0.0/types.yaml @@ -0,0 +1,28 @@ +types: + TestType: u32 +typesAlias: + Alias: + TestType2: test +typesBundle: + spec: + '2312': + types: + - minmax: + - 232 + - 122 + types: + TestType3: test3 + chain: + mockchain: + types: + - minmax: + - 232 + - 122 + types: + TestType4: test4 +typesChain: + chain2: + TestType5: test +typesSpec: + spec3: + TestType6: test diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 53ab65b6c5..6e102b3e84 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Subcommands for `force-clean` and `reindex`. (#1281) +- `yargs` file has been moved back into `node` from `node-core`. (#1281) ## [1.9.2] - 2022-09-19 ### Changed diff --git a/packages/node/README.md b/packages/node/README.md index 348f131d8a..c1431088ae 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -14,6 +14,15 @@ $ npm i -g @subql/node ``` $> subql-node + +Commands: + run force-clean Force cleans the database, dropping project schemas and + tables (Once the command is executed, the application would + exit upon completion) + + run reindex Reindex to specified block height (Once the command is + executed, the application would exit upon completion). + Options: --help Show help [boolean] --version Show version number [boolean] @@ -21,9 +30,7 @@ Options: [string] [required] [default: process.cwd()] --subquery-name Name of the subquery project [deprecated] [string] -c, --config Specify configuration file [string] - --local Use local mode [deprecated] [boolean] - --force-clean Force clean the database, dropping project schemas - and tables [boolean] + --local Use local mode [deprecated] [boolean] [boolean] --db-schema Db schema name of the project [string] --unsafe Allows usage of any built-in module within the sandbox [boolean] @@ -60,7 +67,6 @@ Options: -p, --port The port the service will bind to [number] --disable-historical Disable storing historical state entities [boolean] [default: true] - --reindex Reindex to specified block height [number] -w, --workers Number of worker threads to use for fetching and processing blocks. Disabled by default. [number] ``` diff --git a/packages/node/package.json b/packages/node/package.json index 1c67f25dfc..7100f17949 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@subql/node", - "version": "1.9.3-0", + "version": "1.9.3-3", "description": "", "author": "Ian He", "license": "Apache-2.0", diff --git a/packages/node/src/app.module.ts b/packages/node/src/app.module.ts index 8bdcb17ae2..fe190bd6fd 100644 --- a/packages/node/src/app.module.ts +++ b/packages/node/src/app.module.ts @@ -11,13 +11,7 @@ import { MetaModule } from './meta/meta.module'; @Module({ imports: [ - DbModule.forRoot({ - host: process.env.DB_HOST ?? '127.0.0.1', - port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, - username: process.env.DB_USER ?? 'postgres', - password: process.env.DB_PASS ?? 'postgres', - database: process.env.DB_DATABASE ?? 'postgres', - }), + DbModule.forRoot(), EventEmitterModule.forRoot(), ConfigureModule.register(), ScheduleModule.forRoot(), diff --git a/packages/node/src/configure/configure.module.ts b/packages/node/src/configure/configure.module.ts index 551803c22c..d83dab8c2f 100644 --- a/packages/node/src/configure/configure.module.ts +++ b/packages/node/src/configure/configure.module.ts @@ -10,19 +10,20 @@ import { IConfig, MinConfig, NodeConfig, - getYargsOption, getLogger, setLevel, } from '@subql/node-core'; import { camelCase, last, omitBy, isNil } from 'lodash'; +import { yargsOptions } from '../yargs'; import { SubqueryProject } from './SubqueryProject'; + const logger = getLogger('configure'); -const YargsNameMapping = { +const YargsNameMapping: Record = { local: 'localMode', }; -type Args = ReturnType['argv']; +type Args = typeof yargsOptions.argv['argv']; function yargsToIConfig(yargs: Args): Partial { return Object.entries(yargs).reduce((acc, [key, value]) => { @@ -37,7 +38,7 @@ function yargsToIConfig(yargs: Args): Partial { } acc[YargsNameMapping[key] ?? camelCase(key)] = value; return acc; - }, {}); + }, {} as any); } function defaultSubqueryName(config: Partial): MinConfig { @@ -79,7 +80,6 @@ export function validDbSchemaName(name: string): boolean { } function warnDeprecations() { - const yargsOptions = getYargsOption(); const { argv } = yargsOptions; if (argv['subquery-name']) { logger.warn( @@ -94,8 +94,51 @@ function warnDeprecations() { @Global() @Module({}) export class ConfigureModule { + static registerWithConfig(config: NodeConfig): DynamicModule { + if (!validDbSchemaName(config.dbSchema)) { + process.exit(1); + } + + if (config.debug) { + setLevel('debug'); + } + + const project = async () => { + const p = await SubqueryProject.create( + config.subquery, + omitBy( + { + endpoint: config.networkEndpoint, + dictionary: config.networkDictionary, + }, + isNil, + ), + { + ipfs: config.ipfs, + }, + ).catch((err) => { + logger.error(err, 'Create Subquery project from given path failed!'); + process.exit(1); + }); + return p; + }; + + return { + module: ConfigureModule, + providers: [ + { + provide: NodeConfig, + useValue: config, + }, + { + provide: SubqueryProject, + useFactory: project, + }, + ], + exports: [NodeConfig, SubqueryProject], + }; + } static register(): DynamicModule { - const yargsOptions = getYargsOption(); const { argv } = yargsOptions; let config: NodeConfig; if (argv.config) { diff --git a/packages/node/src/indexer/dictionary.service.ts b/packages/node/src/indexer/dictionary.service.ts index df62e50593..1a297dfcd0 100644 --- a/packages/node/src/indexer/dictionary.service.ts +++ b/packages/node/src/indexer/dictionary.service.ts @@ -9,17 +9,12 @@ import { gql, } from '@apollo/client/core'; import { Injectable, OnApplicationShutdown } from '@nestjs/common'; -import { - getYargsOption, - NodeConfig, - timeout, - getLogger, - profiler, -} from '@subql/node-core'; +import { NodeConfig, timeout, getLogger, profiler } from '@subql/node-core'; import { DictionaryQueryCondition, DictionaryQueryEntry } from '@subql/types'; import { buildQuery, GqlNode, GqlQuery, GqlVar, MetaData } from '@subql/utils'; import fetch from 'node-fetch'; import { SubqueryProject } from '../configure/SubqueryProject'; +import { yargsOptions } from '../yargs'; export type SpecVersion = { id: string; @@ -40,7 +35,6 @@ export type SpecVersionDictionary = { }; const logger = getLogger('dictionary'); -const { argv } = getYargsOption(); function extractVar(name: string, cond: DictionaryQueryCondition): GqlVar { return { @@ -151,7 +145,7 @@ export class DictionaryService implements OnApplicationShutdown { * @param conditions */ - @profiler(argv.profiler) + @profiler(yargsOptions.argv.profiler) async getDictionary( startBlock: number, queryEndBlock: number, diff --git a/packages/node/src/indexer/ds-processor.service.spec.ts b/packages/node/src/indexer/ds-processor.service.spec.ts index fc1bba95ca..f2f3838d0c 100644 --- a/packages/node/src/indexer/ds-processor.service.spec.ts +++ b/packages/node/src/indexer/ds-processor.service.spec.ts @@ -3,6 +3,7 @@ import path from 'path'; import { isCustomDs } from '@subql/common-substrate'; +import { NodeConfig } from '@subql/node-core'; import { SubstrateCustomDatasource } from '@subql/types'; import { GraphQLSchema } from 'graphql'; import { SubqueryProject } from '../configure/SubqueryProject'; @@ -34,6 +35,10 @@ function getTestProject( templates: [], }; } +const nodeConfig = new NodeConfig({ + subquery: 'asdf', + subqueryName: 'asdf', +}); describe('DsProcessorService', () => { let service: DsProcessorService; @@ -41,7 +46,7 @@ describe('DsProcessorService', () => { beforeEach(() => { project = getTestProject([]); - service = new DsProcessorService(project); + service = new DsProcessorService(project, nodeConfig); }); it('can validate custom ds', async () => { @@ -62,7 +67,7 @@ describe('DsProcessorService', () => { }; project = getTestProject([badDs]); - service = new DsProcessorService(project); + service = new DsProcessorService(project, nodeConfig); await expect(service.validateProjectCustomDatasources()).rejects.toThrow(); }); diff --git a/packages/node/src/indexer/ds-processor.service.ts b/packages/node/src/indexer/ds-processor.service.ts index 2784cb655a..38e9ebdcdd 100644 --- a/packages/node/src/indexer/ds-processor.service.ts +++ b/packages/node/src/indexer/ds-processor.service.ts @@ -12,7 +12,7 @@ import { SubstrateDatasourceProcessor, SubstrateNetworkFilter, } from '@subql/common-substrate'; -import { getLogger } from '@subql/node-core'; +import { getLogger, NodeConfig } from '@subql/node-core'; import { SecondLayerHandlerProcessor_0_0_0, SecondLayerHandlerProcessor_1_0_0, @@ -93,13 +93,14 @@ export function asSecondLayerHandlerProcessor_1_0_0< } export class DsPluginSandbox extends Sandbox { - constructor(option: DsPluginSandboxOption) { + constructor(option: DsPluginSandboxOption, nodeConfig: NodeConfig) { super( option, new VMScript( `module.exports = require('${option.entry}').default;`, path.join(option.root, 'ds_sandbox'), ), + nodeConfig, ); this.freeze(logger, 'logger'); } @@ -120,7 +121,10 @@ export class DsProcessorService { SubstrateNetworkFilter >; } = {}; - constructor(private project: SubqueryProject) {} + constructor( + private project: SubqueryProject, + private readonly nodeConfig: NodeConfig, + ) {} async validateCustomDs( datasources: SubstrateCustomDataSource[], @@ -168,11 +172,15 @@ export class DsProcessorService { throw new Error(`data source is not a custom data source`); } if (!this.processorCache[ds.processor.file]) { - const sandbox = new DsPluginSandbox({ - root: this.project.root, - entry: ds.processor.file, - script: null /* TODO get working with Readers, same as with sandbox */, - }); + const sandbox = new DsPluginSandbox( + { + root: this.project.root, + entry: ds.processor.file, + script: + null /* TODO get working with Readers, same as with sandbox */, + }, + this.nodeConfig, + ); try { this.processorCache[ds.processor.file] = sandbox.getDsPlugin(); } catch (e) { diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 4499928020..ea20b17c3c 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -2,13 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import { Module } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; import { BenchmarkService, MmrService, StoreService, PoiService, - getYargsOption, DbModule, + NodeConfig, } from '@subql/node-core'; import { ApiService } from './api.service'; import { DictionaryService } from './dictionary.service'; @@ -23,8 +24,6 @@ import { WorkerBlockDispatcherService, } from './worker/block-dispatcher.service'; -const { argv } = getYargsOption(); - @Module({ imports: [DbModule.forFeature(['Subquery'])], providers: [ @@ -33,9 +32,33 @@ const { argv } = getYargsOption(); IndexerManager, { provide: 'IBlockDispatcher', - useClass: argv.workers - ? WorkerBlockDispatcherService - : BlockDispatcherService, + useFactory: ( + nodeConfig: NodeConfig, + eventEmitter: EventEmitter2, + projectService: ProjectService, + apiService: ApiService, + indexerManager: IndexerManager, + ) => + nodeConfig.workers !== undefined + ? new WorkerBlockDispatcherService( + nodeConfig, + eventEmitter, + projectService, + ) + : new BlockDispatcherService( + apiService, + nodeConfig, + indexerManager, + eventEmitter, + projectService, + ), + inject: [ + NodeConfig, + EventEmitter2, + ProjectService, + ApiService, + IndexerManager, + ], }, FetchService, BenchmarkService, diff --git a/packages/node/src/indexer/fetch.service.spec.ts b/packages/node/src/indexer/fetch.service.spec.ts index 48c0ec1293..e56dfe4a84 100644 --- a/packages/node/src/indexer/fetch.service.spec.ts +++ b/packages/node/src/indexer/fetch.service.spec.ts @@ -277,8 +277,9 @@ function createFetchService( dictionaryService: DictionaryService, project: SubqueryProject, batchSize?: number, + config?: NodeConfig, ) { - const dsProcessorService = new DsProcessorService(project); + const dsProcessorService = new DsProcessorService(project, config); const dynamicDsService = new DynamicDsService(dsProcessorService, project); (dynamicDsService as any).getDynamicDatasources = jest.fn(() => []); const nodeConfig = new NodeConfig({ @@ -311,6 +312,7 @@ describe('FetchService', () => { let apiService: ApiService; let project: SubqueryProject; let fetchService: FetchService; + let config: NodeConfig; beforeEach(() => { apiService = mockApiService(); @@ -446,7 +448,7 @@ describe('FetchService', () => { }); const eventEmitter = new EventEmitter2(); const schedulerRegistry = new SchedulerRegistry(); - const dsProcessorService = new DsProcessorService(project); + const dsProcessorService = new DsProcessorService(project, config); const dynamicDsService = new DynamicDsService(dsProcessorService, project); (dynamicDsService as any).getDynamicDatasources = jest.fn(() => []); const nodeConfig = new NodeConfig({ @@ -530,7 +532,7 @@ describe('FetchService', () => { const dictionaryService = mockDictionaryService3(); const schedulerRegistry = new SchedulerRegistry(); const eventEmitter = new EventEmitter2(); - const dsProcessorService = new DsProcessorService(project); + const dsProcessorService = new DsProcessorService(project, config); const dynamicDsService = new DynamicDsService(dsProcessorService, project); (dynamicDsService as any).getDynamicDatasources = jest.fn(() => []); const nodeConfig = new NodeConfig({ @@ -607,7 +609,7 @@ describe('FetchService', () => { ]; const dictionaryService = mockDictionaryService1(); const schedulerRegistry = new SchedulerRegistry(); - const dsProcessorService = new DsProcessorService(project); + const dsProcessorService = new DsProcessorService(project, config); const dynamicDsService = new DynamicDsService(dsProcessorService, project); (dynamicDsService as any).getDynamicDatasources = jest.fn(() => []); const eventEmitter = new EventEmitter2(); @@ -668,6 +670,7 @@ describe('FetchService', () => { new DictionaryService(project, nodeConfig), project, 20, + nodeConfig, ); const baseHandlerFilters = jest.spyOn( diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 8620c64c35..5af758551a 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -26,7 +26,6 @@ import { checkMemoryUsage, NodeConfig, IndexerEvent, - getYargsOption, getLogger, profiler, } from '@subql/node-core'; @@ -41,6 +40,7 @@ import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; import { isBaseHandler, isCustomHandler } from '../utils/project'; import * as SubstrateUtil from '../utils/substrate'; import { calcInterval } from '../utils/substrate'; +import { yargsOptions } from '../yargs'; import { ApiService } from './api.service'; import { DictionaryService, SpecVersion } from './dictionary.service'; import { DsProcessorService } from './ds-processor.service'; @@ -55,8 +55,6 @@ const MINIMUM_BATCH_SIZE = 5; const SPEC_VERSION_BLOCK_GAP = 100; const INTERVAL_PERCENT = 0.9; -const { argv } = getYargsOption(); - function eventFilterToQueryEntry( filter: SubstrateEventFilter, ): DictionaryQueryEntry { @@ -276,8 +274,8 @@ export class FetchService implements OnApplicationShutdown { @Interval(CHECK_MEMORY_INTERVAL) checkBatchScale(): void { - if (argv['scale-batch-size']) { - const scale = checkMemoryUsage(this.batchSizeScale); + if (this.nodeConfig['scale-batch-size']) { + const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig); if (this.batchSizeScale !== scale) { this.batchSizeScale = scale; @@ -507,7 +505,7 @@ export class FetchService implements OnApplicationShutdown { return this.currentRuntimeVersion; } - @profiler(argv.profiler) + @profiler(yargsOptions.argv.profiler) async specChanged(height: number): Promise { const specVersion = await this.getSpecVersion(height); if (this.parentSpecVersion !== specVersion) { @@ -518,7 +516,7 @@ export class FetchService implements OnApplicationShutdown { return false; } - @profiler(argv.profiler) + @profiler(yargsOptions.argv.profiler) async prefetchMeta(height: number): Promise { const blockHash = await this.api.rpc.chain.getBlockHash(height); if ( diff --git a/packages/node/src/indexer/indexer.manager.spec.ts b/packages/node/src/indexer/indexer.manager.spec.ts index f997d3efd6..e224a4bdd3 100644 --- a/packages/node/src/indexer/indexer.manager.spec.ts +++ b/packages/node/src/indexer/indexer.manager.spec.ts @@ -124,12 +124,14 @@ function testSubqueryProject_2(): SubqueryProject { }; } -function createIndexerManager(project: SubqueryProject): IndexerManager { +function createIndexerManager( + project: SubqueryProject, + nodeConfig: NodeConfig, +): IndexerManager { const sequilize = new Sequelize(); const eventEmitter = new EventEmitter2(); - const apiService = new ApiService(project, eventEmitter); - const dsProcessorService = new DsProcessorService(project); + const dsProcessorService = new DsProcessorService(project, nodeConfig); const dynamicDsService = new DynamicDsService(dsProcessorService, project); const poiService = new PoiService(nodeConfig, sequilize); @@ -182,14 +184,14 @@ describe('IndexerManager', () => { }); xit('should be able to start the manager (v0.0.1)', async () => { - indexerManager = createIndexerManager(testSubqueryProject_1()); + indexerManager = createIndexerManager(testSubqueryProject_1(), nodeConfig); await expect(indexerManager.start()).resolves.toBe(undefined); expect(Object.keys((indexerManager as any).vms).length).toBe(1); }); xit('should be able to start the manager (v0.2.0)', async () => { - indexerManager = createIndexerManager(testSubqueryProject_2()); + indexerManager = createIndexerManager(testSubqueryProject_2(), nodeConfig); await expect(indexerManager.start()).resolves.toBe(undefined); expect(Object.keys((indexerManager as any).vms).length).toBe(1); diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index 565c36dce6..96e385fc4c 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -23,7 +23,6 @@ import { PoiService, SubqueryRepo, NodeConfig, - getYargsOption, getLogger, profiler, profilerWrap, @@ -36,6 +35,7 @@ import { import { Sequelize } from 'sequelize'; import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; import * as SubstrateUtil from '../utils/substrate'; +import { yargsOptions } from '../yargs'; import { ApiService } from './api.service'; import { asSecondLayerHandlerProcessor_1_0_0, @@ -49,7 +49,6 @@ import { ApiAt, BlockContent } from './types'; const NULL_MERKEL_ROOT = hexToU8a('0x00'); const logger = getLogger('indexer'); -const { argv } = getYargsOption(); @Injectable() export class IndexerManager { @@ -70,10 +69,11 @@ export class IndexerManager { private projectService: ProjectService, ) { logger.info('indexer manager start'); + this.api = this.apiService.getApi(); } - @profiler(argv.profiler) + @profiler(yargsOptions.argv.profiler) async indexBlock( blockContent: BlockContent, runtimeVersion: RuntimeVersion, @@ -284,7 +284,7 @@ export class IndexerManager { for (const handler of handlers) { vm = vm ?? (await getVM(ds)); - argv.profiler + this.nodeConfig.profiler ? await profilerWrap( vm.securedExec.bind(vm), 'handlerPerformance', @@ -337,6 +337,7 @@ export class IndexerManager { ) => boolean, ): SubstrateCustomHandler[] { const plugin = this.dsProcessorService.getDsProcessor(ds); + return ds.mapping.handlers .filter((handler) => { const processor = plugin.handlerProcessors[handler.kind]; @@ -350,6 +351,7 @@ export class IndexerManager { const processor = asSecondLayerHandlerProcessor_1_0_0( plugin.handlerProcessors[handler.kind], ); + try { return processor.filterProcessor({ filter: handler.filter, @@ -371,6 +373,7 @@ export class IndexerManager { ): Promise { const plugin = this.dsProcessorService.getDsProcessor(ds); const assets = await this.dsProcessorService.getAssets(ds); + const processor = asSecondLayerHandlerProcessor_1_0_0( plugin.handlerProcessors[handler.kind], ); diff --git a/packages/node/src/indexer/project.service.test.ts b/packages/node/src/indexer/project.service.test.ts index 157ea594a1..59aca40f8c 100644 --- a/packages/node/src/indexer/project.service.test.ts +++ b/packages/node/src/indexer/project.service.test.ts @@ -2,9 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 import { Test } from '@nestjs/testing'; -import { SubqueryRepo, DbModule, NodeConfig } from '@subql/node-core'; +import { + SubqueryRepo, + DbModule, + NodeConfig, + getExistingProjectSchema, +} from '@subql/node-core'; import { GraphQLSchema } from 'graphql'; import { Sequelize } from 'sequelize'; +import { ConfigureModule } from '../configure/configure.module'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ProjectService } from './project.service'; @@ -21,6 +27,11 @@ function testSubqueryProject(): SubqueryProject { templates: [], }; } +const TEST_PROJECT = 'test-user/TEST_PROJECT'; +const nodeConfig = new NodeConfig({ + subquery: 'packages/node/test/projectFixture/v1.0.0', + subqueryName: TEST_PROJECT, +}); const prepare = async (): Promise => { const module = await Test.createTestingModule({ @@ -35,8 +46,9 @@ const prepare = async (): Promise => { sequelize: Sequelize, project: SubqueryProject, subqueryRepo: SubqueryRepo, - ) => { - const projectService = new ProjectService( + nodeConfig: NodeConfig, + ) => + new ProjectService( undefined, undefined, undefined, @@ -44,25 +56,17 @@ const prepare = async (): Promise => { sequelize, project, undefined, - undefined, + nodeConfig, undefined, subqueryRepo, undefined, - ); - - return projectService; - }, - inject: [Sequelize, SubqueryProject, 'Subquery'], + ), + inject: [Sequelize, SubqueryProject, 'Subquery', NodeConfig], }, ], imports: [ - DbModule.forRoot({ - host: process.env.DB_HOST ?? '127.0.0.1', - port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, - username: process.env.DB_USER ?? 'postgres', - password: process.env.DB_PASS ?? 'postgres', - database: process.env.DB_DATABASE ?? 'postgres', - }), + ConfigureModule.registerWithConfig(nodeConfig), + DbModule.forRoot(), DbModule.forFeature(['Subquery']), ], }).compile(); @@ -87,11 +91,10 @@ function prepareProject( }; } -const TEST_PROJECT = 'test-user/TEST_PROJECT'; - describe('ProjectService Integration Tests', () => { let projectService: ProjectService; let subqueryRepo: SubqueryRepo; + let logger: any; async function createSchema(name: string): Promise { await subqueryRepo.sequelize.createSchema(`"${name}"`, undefined); @@ -108,37 +111,40 @@ describe('ProjectService Integration Tests', () => { }); beforeEach(async () => { - delete (projectService as any).nodeConfig; await subqueryRepo.destroy({ where: { name: TEST_PROJECT } }); await subqueryRepo.sequelize.dropSchema(`"${TEST_PROJECT}"`, undefined); }); it("read existing project's schema from subqueries table", async () => { const schemaName = 'subql_99999'; - (projectService as any).nodeConfig = new NodeConfig({ - subquery: '/test/dir/test-query-project', - subqueryName: TEST_PROJECT, - }); - await subqueryRepo.create(prepareProject(TEST_PROJECT, schemaName, 1)); - await expect( - (projectService as any).getExistingProjectSchema(), - ).resolves.toBe(schemaName); + const schema = getExistingProjectSchema( + (projectService as any).nodeConfig, + (projectService as any).sequelize, + (projectService as any).subqueryRepo, + ); + await expect(schema).resolves.toBe(schemaName); }); it("read existing project's schema from nodeConfig", async () => { - (projectService as any).nodeConfig = new NodeConfig({ - subquery: '/test/dir/test-query-project', - subqueryName: TEST_PROJECT, - }); - await createSchema(TEST_PROJECT); await subqueryRepo.create(prepareProject(TEST_PROJECT, 'subql_99999', 1)); - await expect( - (projectService as any).getExistingProjectSchema(), - ).resolves.toBe(TEST_PROJECT); + const schema = getExistingProjectSchema( + (projectService as any).nodeConfig, + (projectService as any).sequelize, + (projectService as any).subqueryRepo, + ); + + await expect(schema).resolves.toBe(TEST_PROJECT); + }); + + it('create project schema', async () => { + await expect((projectService as any).createProjectSchema()).resolves.toBe( + TEST_PROJECT, + ); + await expect(checkSchemaExist(TEST_PROJECT)).resolves.toBe(true); }); it("read existing project's schema when --local", async () => { @@ -150,19 +156,12 @@ describe('ProjectService Integration Tests', () => { await createSchema(TEST_PROJECT); await subqueryRepo.create(prepareProject(TEST_PROJECT, 'subql_99999', 1)); - await expect( - (projectService as any).getExistingProjectSchema(), - ).resolves.toBe('public'); - }); - - it('create project schema', async () => { - (projectService as any).nodeConfig = new NodeConfig({ - subquery: '/test/dir/test-query-project', - subqueryName: TEST_PROJECT, - }); - await expect((projectService as any).createProjectSchema()).resolves.toBe( - TEST_PROJECT, + const schema = getExistingProjectSchema( + (projectService as any).nodeConfig, + (projectService as any).sequelize, + (projectService as any).subqueryRepo, ); - await expect(checkSchemaExist(TEST_PROJECT)).resolves.toBe(true); + + await expect(schema).resolves.toBe('public'); }); }); diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 40615d420c..1d6a9a74c4 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import assert from 'assert'; -import fs from 'fs'; import { isMainThread } from 'worker_threads'; import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; @@ -15,12 +14,13 @@ import { StoreService, PoiService, MmrService, - getYargsOption, getLogger, + getExistingProjectSchema, + getMetaDataInfo, } from '@subql/node-core'; -import { getAllEntitiesRelations } from '@subql/utils'; -import { QueryTypes, Sequelize } from 'sequelize'; +import { Sequelize } from 'sequelize'; import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; +import { initDbSchema } from '../utils/project'; import { ApiService } from './api.service'; import { DsProcessorService } from './ds-processor.service'; import { DynamicDsService } from './dynamic-ds.service'; @@ -31,7 +31,6 @@ const { version: packageVersion } = require('../../package.json'); const DEFAULT_DB_SCHEMA = 'public'; const logger = getLogger('Project'); -const { argv } = getYargsOption(); @Injectable() export class ProjectService { @@ -66,6 +65,14 @@ export class ProjectService { return this._startHeight; } + private async getExistingProjectSchema(): Promise { + return getExistingProjectSchema( + this.nodeConfig, + this.sequelize, + this.subqueryRepo, + ); + } + async init(): Promise { // Do extra work on main thread to setup stuff if (isMainThread) { @@ -83,9 +90,6 @@ export class ProjectService { } this._startHeight = await this.getStartHeight(); - if (argv.reindex !== undefined) { - await this.reindex(argv.reindex); - } } else { this.metadataRepo = MetadataFactory(this.sequelize, this.schema); @@ -107,39 +111,7 @@ export class ProjectService { let schema = await this.getExistingProjectSchema(); if (!schema) { schema = await this.createProjectSchema(); - } else { - if (argv['force-clean']) { - try { - // drop existing project schema and metadata table - await this.sequelize.dropSchema(`"${schema}"`, { - logging: false, - benchmark: false, - }); - - // remove schema from subquery table (might not exist) - await this.sequelize.query( - ` DELETE - FROM public.subqueries - WHERE name = :name`, - { - replacements: { name: this.nodeConfig.subqueryName }, - type: QueryTypes.DELETE, - }, - ); - - logger.info('force cleaned schema and tables'); - - if (fs.existsSync(this.nodeConfig.mmrPath)) { - await fs.promises.unlink(this.nodeConfig.mmrPath); - logger.info('force cleaned file based mmr'); - } - } catch (err) { - logger.error(err, 'failed to force clean'); - } - schema = await this.createProjectSchema(); - } } - this.eventEmitter.emit(IndexerEvent.Ready, { value: true, }); @@ -147,37 +119,6 @@ export class ProjectService { return schema; } - // Get existing project schema, undefined when doesn't exist - private async getExistingProjectSchema(): Promise { - let schema = this.nodeConfig.localMode - ? DEFAULT_DB_SCHEMA - : this.nodeConfig.dbSchema; - - // Note that sequelize.fetchAllSchemas does not include public schema, we cannot assume that public schema exists so we must make a raw query - const schemas = (await this.sequelize - .query(`SELECT schema_name FROM information_schema.schemata`, { - type: QueryTypes.SELECT, - }) - .then((xs) => xs.map((x: any) => x.schema_name)) - .catch((err) => { - logger.error(`Unable to fetch all schemas: ${err}`); - process.exit(1); - })) as [string]; - - if (!schemas.includes(schema)) { - // fallback to subqueries table - const subqueryModel = await this.subqueryRepo.findOne({ - where: { name: this.nodeConfig.subqueryName }, - }); - if (subqueryModel) { - schema = subqueryModel.dbSchema; - } else { - schema = undefined; - } - } - return schema; - } - private async createProjectSchema(): Promise { let schema: string; if (this.nodeConfig.localMode) { @@ -195,9 +136,7 @@ export class ProjectService { } private async initDbSchema(): Promise { - const graphqlSchema = this.project.schema; - const modelsRelations = getAllEntitiesRelations(graphqlSchema); - await this.storeService.init(modelsRelations, this.schema); + await initDbSchema(this.project, this.schema, this.storeService); } private async ensureMetadata(): Promise { @@ -305,19 +244,11 @@ export class ProjectService { } async getMetadataBlockOffset(): Promise { - const res = await this.metadataRepo.findOne({ - where: { key: 'blockOffset' }, - }); - - return res?.value as number | undefined; + return getMetaDataInfo(this.metadataRepo, 'blockOffset'); } async getLastProcessedHeight(): Promise { - const res = await this.metadataRepo.findOne({ - where: { key: 'lastProcessedHeight' }, - }); - - return res?.value as number | undefined; + return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight'); } private async getStartHeight(): Promise { @@ -387,33 +318,4 @@ export class ProjectService { this.apiService.getApi().runtimeVersion.specName.toString(), ); } - - private async reindex(targetBlockHeight: number): Promise { - const lastProcessedHeight = await this.getLastProcessedHeight(); - if (!this.storeService.historical) { - logger.warn('Unable to reindex, historical state not enabled'); - return; - } - if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) { - logger.warn( - `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`, - ); - return; - } - logger.info(`Reindexing to block: ${targetBlockHeight}`); - const transaction = await this.sequelize.transaction(); - try { - await this.storeService.rewind(argv.reindex, transaction); - - const blockOffset = await this.getMetadataBlockOffset(); - if (blockOffset) { - await this.mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset); - } - await transaction.commit(); - } catch (err) { - logger.error(err, 'Reindexing failed'); - await transaction.rollback(); - throw err; - } - } } diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts index 4e870ce6cf..5521d0d9cb 100644 --- a/packages/node/src/indexer/sandbox.service.ts +++ b/packages/node/src/indexer/sandbox.service.ts @@ -7,13 +7,7 @@ import { isDatasourceV0_2_0, SubstrateDataSource, } from '@subql/common-substrate'; -import { - timeout, - NodeConfig, - StoreService, - getYargsOption, - getLogger, -} from '@subql/node-core'; +import { timeout, NodeConfig, StoreService, getLogger } from '@subql/node-core'; import { Store } from '@subql/types'; import { levelFilter } from '@subql/utils'; import { merge } from 'lodash'; @@ -23,8 +17,6 @@ import { getProjectEntry } from '../utils/project'; import { ApiService } from './api.service'; import { ApiAt } from './types'; -const { argv } = getYargsOption(); - export interface SandboxOption { store?: Store; script: string; @@ -32,27 +24,33 @@ export interface SandboxOption { entry: string; } -const DEFAULT_OPTION: NodeVMOptions = { - console: 'redirect', - wasm: argv.unsafe, - sandbox: {}, - require: { - builtin: argv.unsafe - ? ['*'] - : ['assert', 'buffer', 'crypto', 'util', 'path'], - external: true, - context: 'sandbox', - }, - wrapper: 'commonjs', - sourceExtensions: ['js', 'cjs'], +const DEFAULT_OPTION = (nodeConfig: NodeConfig): NodeVMOptions => { + return { + console: 'redirect', + wasm: nodeConfig?.unsafe, + sandbox: {}, + require: { + builtin: nodeConfig.unsafe + ? ['*'] + : ['assert', 'buffer', 'crypto', 'util', 'path'], + external: true, + context: 'sandbox', + }, + wrapper: 'commonjs', + sourceExtensions: ['js', 'cjs'], + }; }; const logger = getLogger('sandbox'); export class Sandbox extends NodeVM { - constructor(option: SandboxOption, protected readonly script: VMScript) { + constructor( + option: SandboxOption, + protected readonly script: VMScript, + protected config: NodeConfig, + ) { super( - merge(DEFAULT_OPTION, { + merge(DEFAULT_OPTION(config), { require: { root: option.root, resolve: (moduleName: string) => { @@ -69,7 +67,7 @@ export class Sandbox extends NodeVM { } export class IndexerSandbox extends Sandbox { - constructor(option: SandboxOption, private readonly config: NodeConfig) { + constructor(option: SandboxOption, config: NodeConfig) { super( option, new VMScript( @@ -78,6 +76,7 @@ export class IndexerSandbox extends Sandbox { `, path.join(option.root, 'sandbox'), ), + config, ); this.injectGlobals(option); } @@ -135,7 +134,7 @@ export class SandboxService { this.processorCache[entry] = processor; } processor.freeze(api, 'api'); - if (argv.unsafe) { + if (this.nodeConfig.unsafe) { processor.freeze(this.apiService.getApi(), 'unsafeApi'); } return processor; diff --git a/packages/node/src/indexer/worker/block-dispatcher.service.ts b/packages/node/src/indexer/worker/block-dispatcher.service.ts index df699fadbe..f64b9a3f9d 100644 --- a/packages/node/src/indexer/worker/block-dispatcher.service.ts +++ b/packages/node/src/indexer/worker/block-dispatcher.service.ts @@ -14,7 +14,6 @@ import { IndexerEvent, Worker, delay, - getYargsOption, profilerWrap, AutoQueue, Queue, @@ -129,9 +128,7 @@ export class BlockDispatcherService this.fetchQueue = new Queue(nodeConfig.batchSize * 3); this.processQueue = new AutoQueue(nodeConfig.batchSize * 3); - const { argv } = getYargsOption(); - - if (argv.profiler) { + if (this.nodeConfig.profiler) { this.fetchBlocksBatches = profilerWrap( SubstrateUtil.fetchBlocksBatches, 'SubstrateUtil', diff --git a/packages/node/src/indexer/worker/worker.module.ts b/packages/node/src/indexer/worker/worker.module.ts index f2f0e681ff..8f26dda25c 100644 --- a/packages/node/src/indexer/worker/worker.module.ts +++ b/packages/node/src/indexer/worker/worker.module.ts @@ -10,13 +10,7 @@ import { IndexerModule } from '../indexer.module'; @Module({ imports: [ - DbModule.forRoot({ - host: process.env.DB_HOST ?? '127.0.0.1', - port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, - username: process.env.DB_USER ?? 'postgres', - password: process.env.DB_PASS ?? 'postgres', - database: process.env.DB_DATABASE ?? 'postgres', - }), + DbModule.forRoot(), EventEmitterModule.forRoot(), ConfigureModule.register(), ScheduleModule.forRoot(), diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts new file mode 100644 index 0000000000..43972f1be9 --- /dev/null +++ b/packages/node/src/init.ts @@ -0,0 +1,64 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import { NestFactory } from '@nestjs/core'; +import { findAvailablePort } from '@subql/common'; +import { getLogger, NestLogger } from '@subql/node-core'; +import { AppModule } from './app.module'; +import { ApiService } from './indexer/api.service'; +import { FetchService } from './indexer/fetch.service'; +import { ProjectService } from './indexer/project.service'; +import { yargsOptions } from './yargs'; + +const { argv } = yargsOptions; + +const DEFAULT_PORT = 3000; +const logger = getLogger('subql-node'); + +export async function bootstrap() { + const debug = argv.debug; + + const validate = (x: any) => { + const p = parseInt(x); + return isNaN(p) ? null : p; + }; + + const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT)); + if (!port) { + logger.error( + `Unable to find available port (tried ports in range (${port}..${ + port + 10 + })). Try setting a free port manually by setting the --port flag`, + ); + process.exit(1); + } + + if (argv.unsafe) { + logger.warn( + 'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service', + ); + } + + try { + const app = await NestFactory.create(AppModule, { + logger: debug ? new NestLogger() : false, + }); + await app.init(); + + const projectService = app.get(ProjectService); + const fetchService = app.get(FetchService); + const apiService = app.get(ApiService); + + // Initialise async services, we do this here rather than in factories, so we can capture one off events + await apiService.init(); + await projectService.init(); + await fetchService.init(projectService.startHeight); + + await app.listen(port); + + logger.info(`Node started on port: ${port}`); + } catch (e) { + logger.error(e, 'Node failed to start'); + process.exit(1); + } +} diff --git a/packages/node/src/main.ts b/packages/node/src/main.ts index cb5472317d..913088af2d 100644 --- a/packages/node/src/main.ts +++ b/packages/node/src/main.ts @@ -1,64 +1,19 @@ // Copyright 2020-2022 OnFinality Limited authors & contributors // SPDX-License-Identifier: Apache-2.0 -import { NestFactory } from '@nestjs/core'; -import { findAvailablePort } from '@subql/common'; -import { getYargsOption, getLogger, NestLogger } from '@subql/node-core'; -import { AppModule } from './app.module'; -import { ApiService } from './indexer/api.service'; -import { FetchService } from './indexer/fetch.service'; -import { ProjectService } from './indexer/project.service'; +import { initLogger } from '@subql/node-core/logger'; +import { yargsOptions } from './yargs'; -const DEFAULT_PORT = 3000; -const logger = getLogger('subql-node'); -const { argv } = getYargsOption(); +const { argv } = yargsOptions; -async function bootstrap() { - const debug = argv.debug; - - const validate = (x: any) => { - const p = parseInt(x); - return isNaN(p) ? null : p; - }; - - const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT)); - if (!port) { - logger.error( - `Unable to find available port (tried ports in range (${port}..${ - port + 10 - })). Try setting a free port manually by setting the --port flag`, - ); - process.exit(1); - } - - if (argv.unsafe) { - logger.warn( - 'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service', - ); - } - - try { - const app = await NestFactory.create(AppModule, { - logger: debug ? new NestLogger() : false, - }); - await app.init(); - - const projectService = app.get(ProjectService); - const fetchService = app.get(FetchService); - const apiService = app.get(ApiService); - - // Initialise async services, we do this here rather than in factories so we can capture one off events - await apiService.init(); - await projectService.init(); - await fetchService.init(projectService.startHeight); - - await app.listen(port); - - logger.info(`Node started on port: ${port}`); - } catch (e) { - logger.error(e, 'Node failed to start'); - process.exit(1); - } -} +// initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised +initLogger( + argv.debug, + argv.outputFormat as 'json' | 'colored', + argv.logLevel as string | undefined, +); +// Lazy import, to allow logger to be initialised before bootstrap() +// As bootstrap runs services that requires logger +const { bootstrap } = require('./init'); void bootstrap(); diff --git a/packages/node/src/subcommands/forceClean.init.ts b/packages/node/src/subcommands/forceClean.init.ts new file mode 100644 index 0000000000..1765c00959 --- /dev/null +++ b/packages/node/src/subcommands/forceClean.init.ts @@ -0,0 +1,22 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import { NestFactory } from '@nestjs/core'; +import { getLogger } from '@subql/node-core'; +import { ForceCleanModule } from './forceClean.module'; +import { ForceCleanService } from './forceClean.service'; + +const logger = getLogger('CLI'); +export async function forceCleanInit(): Promise { + try { + const app = await NestFactory.create(ForceCleanModule); + await app.init(); + const forceCleanService = app.get(ForceCleanService); + await forceCleanService.forceClean(); + } catch (e) { + logger.error(e, 'Force-clean failed to execute'); + process.exit(1); + } + + process.exit(0); +} diff --git a/packages/node/src/subcommands/forceClean.module.ts b/packages/node/src/subcommands/forceClean.module.ts new file mode 100644 index 0000000000..40048522a3 --- /dev/null +++ b/packages/node/src/subcommands/forceClean.module.ts @@ -0,0 +1,24 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import { Module } from '@nestjs/common'; +import { DbModule } from '@subql/node-core'; +import { ConfigureModule } from '../configure/configure.module'; +import { ForceCleanService } from './forceClean.service'; + +@Module({ + imports: [DbModule.forFeature(['Subquery'])], + providers: [ForceCleanService], + controllers: [], +}) +export class ForceCleanFeatureModule {} + +@Module({ + imports: [ + DbModule.forRoot(), + ConfigureModule.register(), + ForceCleanFeatureModule, + ], + controllers: [], +}) +export class ForceCleanModule {} diff --git a/packages/node/src/subcommands/forceClean.service.ts b/packages/node/src/subcommands/forceClean.service.ts new file mode 100644 index 0000000000..3785e7e28a --- /dev/null +++ b/packages/node/src/subcommands/forceClean.service.ts @@ -0,0 +1,64 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import fs from 'fs'; +import { Inject, Injectable } from '@nestjs/common'; +import { + getLogger, + NodeConfig, + SubqueryRepo, + getExistingProjectSchema, +} from '@subql/node-core'; +import { QueryTypes, Sequelize } from 'sequelize'; + +const logger = getLogger('Force-clean'); + +@Injectable() +export class ForceCleanService { + constructor( + private readonly sequelize: Sequelize, + private readonly nodeConfig: NodeConfig, + @Inject('Subquery') protected subqueryRepo: SubqueryRepo, + ) {} + + async forceClean(): Promise { + const schema = await getExistingProjectSchema( + this.nodeConfig, + this.sequelize, + this.subqueryRepo, + ); + if (!schema) { + logger.error('Unable to locate schema'); + throw new Error('Schema does not exist.'); + } + + try { + // drop existing project schema and metadata table + await this.sequelize.dropSchema(`"${schema}"`, { + logging: false, + benchmark: false, + }); + + // remove schema from subquery table (might not exist) + await this.sequelize.query( + ` DELETE + FROM public.subqueries + WHERE name = :name`, + { + replacements: { name: this.nodeConfig.subqueryName }, + type: QueryTypes.DELETE, + }, + ); + + logger.info('force cleaned schema and tables'); + + if (fs.existsSync(this.nodeConfig.mmrPath)) { + await fs.promises.unlink(this.nodeConfig.mmrPath); + logger.info('force cleaned file based mmr'); + } + } catch (err) { + logger.error(err, 'failed to force clean'); + throw err; + } + } +} diff --git a/packages/node/src/subcommands/reindex.init.ts b/packages/node/src/subcommands/reindex.init.ts new file mode 100644 index 0000000000..563675c037 --- /dev/null +++ b/packages/node/src/subcommands/reindex.init.ts @@ -0,0 +1,22 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import { NestFactory } from '@nestjs/core'; +import { getLogger } from '@subql/node-core'; +import { ReindexModule } from './reindex.module'; +import { ReindexService } from './reindex.service'; + +const logger = getLogger('CLI-Reindex'); +export async function reindexInit(targetHeight: number): Promise { + try { + const app = await NestFactory.create(ReindexModule); + + await app.init(); + const reindexService = app.get(ReindexService); + await reindexService.reindex(targetHeight); + } catch (e) { + logger.error(e, 'Reindex failed to execute'); + process.exit(1); + } + process.exit(0); +} diff --git a/packages/node/src/subcommands/reindex.module.ts b/packages/node/src/subcommands/reindex.module.ts new file mode 100644 index 0000000000..773eace860 --- /dev/null +++ b/packages/node/src/subcommands/reindex.module.ts @@ -0,0 +1,25 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import { Module } from '@nestjs/common'; +import { DbModule, MmrService, StoreService } from '@subql/node-core'; +import { ConfigureModule } from '../configure/configure.module'; +import { ForceCleanService } from './forceClean.service'; +import { ReindexService } from './reindex.service'; + +@Module({ + imports: [DbModule.forFeature(['Subquery'])], + providers: [StoreService, ReindexService, MmrService, ForceCleanService], + controllers: [], +}) +export class ReindexFeatureModule {} + +@Module({ + imports: [ + DbModule.forRoot(), + ConfigureModule.register(), + ReindexFeatureModule, + ], + controllers: [], +}) +export class ReindexModule {} diff --git a/packages/node/src/subcommands/reindex.service.ts b/packages/node/src/subcommands/reindex.service.ts new file mode 100644 index 0000000000..39aebe83b4 --- /dev/null +++ b/packages/node/src/subcommands/reindex.service.ts @@ -0,0 +1,142 @@ +// Copyright 2020-2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import { Inject, Injectable } from '@nestjs/common'; +import { + getLogger, + MetadataFactory, + MetadataRepo, + MmrService, + NodeConfig, + StoreService, + SubqueryRepo, + getExistingProjectSchema, + getMetaDataInfo, +} from '@subql/node-core'; +import { Sequelize } from 'sequelize'; +import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; +import { initDbSchema } from '../utils/project'; + +import { ForceCleanService } from './forceClean.service'; + +const logger = getLogger('Reindex'); + +@Injectable() +export class ReindexService { + private schema: string; + private metadataRepo: MetadataRepo; + private specName: string; + private startHeight: number; + constructor( + private readonly sequelize: Sequelize, + private readonly nodeConfig: NodeConfig, + private readonly storeService: StoreService, + private readonly mmrService: MmrService, + private readonly project: SubqueryProject, + private readonly forceCleanService: ForceCleanService, + @Inject('Subquery') protected subqueryRepo: SubqueryRepo, + ) {} + + private async getExistingProjectSchema(): Promise { + return getExistingProjectSchema( + this.nodeConfig, + this.sequelize, + this.subqueryRepo, + ); + } + + private async getLastProcessedHeight(): Promise { + return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight'); + } + + private async getMetadataBlockOffset(): Promise { + return getMetaDataInfo(this.metadataRepo, 'blockOffset'); + } + + private async getMetadataSpecName(): Promise { + const res = await this.metadataRepo.findOne({ + where: { key: 'specName' }, + }); + return res?.value as string | undefined; + } + + private async initDbSchema(): Promise { + await initDbSchema(this.project, this.schema, this.storeService); + } + + private async getDataSourcesForSpecName(): Promise { + this.specName = await this.getMetadataSpecName(); + return this.project.dataSources.filter( + (ds) => !ds.filter?.specName || ds.filter.specName === this.specName, + ); + } + + private async getStartBlockFromDataSources() { + const datasources = await this.getDataSourcesForSpecName(); + + const startBlocksList = datasources.map((item) => item.startBlock ?? 1); + if (startBlocksList.length === 0) { + logger.error( + `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`, + ); + process.exit(1); + } else { + return Math.min(...startBlocksList); + } + } + + async reindex(targetBlockHeight: number): Promise { + this.schema = await this.getExistingProjectSchema(); + + if (!this.schema) { + logger.error('Unable to locate schema'); + throw new Error('Schema does not exist.'); + } + await this.initDbSchema(); + + this.metadataRepo = MetadataFactory(this.sequelize, this.schema); + + this.startHeight = await this.getStartBlockFromDataSources(); + + const lastProcessedHeight = await this.getLastProcessedHeight(); + + if (!this.storeService.historical) { + logger.warn('Unable to reindex, historical state not enabled'); + return; + } + if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) { + logger.warn( + `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`, + ); + return; + } + + // if startHeight is greater than the targetHeight, just force clean + if (targetBlockHeight < this.startHeight) { + logger.info( + `targetHeight: ${targetBlockHeight} is less than startHeight: ${this.startHeight}. Hence executing force-clean`, + ); + await this.forceCleanService.forceClean(); + } else { + logger.info(`Reindexing to block: ${targetBlockHeight}`); + const transaction = await this.sequelize.transaction(); + try { + await this.storeService.rewind(targetBlockHeight, transaction); + + const blockOffset = await this.getMetadataBlockOffset(); + if (blockOffset) { + await this.mmrService.deleteMmrNode( + targetBlockHeight + 1, + blockOffset, + ); + } + await transaction.commit(); + logger.info('Reindex Success'); + } catch (err) { + logger.error(err, 'Reindexing failed'); + await transaction.rollback(); + throw err; + } + } + } +} diff --git a/packages/node/src/utils/project.ts b/packages/node/src/utils/project.ts index b80045cfdf..fc1ac234cb 100644 --- a/packages/node/src/utils/project.ts +++ b/packages/node/src/utils/project.ts @@ -24,10 +24,12 @@ import { SubstrateHandler, SubstrateHandlerKind, } from '@subql/common-substrate'; +import { StoreService } from '@subql/node-core'; +import { getAllEntitiesRelations } from '@subql/utils'; import yaml from 'js-yaml'; import tar from 'tar'; import { NodeVM, VMScript } from 'vm2'; -import { SubqlProjectDs } from '../configure/SubqueryProject'; +import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; export async function prepareProjectDir(projectPath: string): Promise { const stats = fs.statSync(projectPath); @@ -299,3 +301,12 @@ export function loadChainTypesFromJs( } return rawContent; } + +export async function initDbSchema( + project: SubqueryProject, + schema: string, + storeService: StoreService, +): Promise { + const modelsRelation = getAllEntitiesRelations(project.schema); + await storeService.init(modelsRelation, schema); +} diff --git a/packages/node-core/src/yargs.ts b/packages/node/src/yargs.ts similarity index 66% rename from packages/node-core/src/yargs.ts rename to packages/node/src/yargs.ts index b5ef41390b..1cd8cde929 100644 --- a/packages/node-core/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -1,12 +1,52 @@ // Copyright 2020-2022 OnFinality Limited authors & contributors // SPDX-License-Identifier: Apache-2.0 -import {hideBin} from 'yargs/helpers'; +import { initLogger } from '@subql/node-core/logger'; +import { hideBin } from 'yargs/helpers'; import yargs from 'yargs/yargs'; -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types -export function getYargsOption() { - return yargs(hideBin(process.argv)).options({ +export const yargsOptions = yargs(hideBin(process.argv)) + .command({ + command: 'force-clean', + describe: + 'Clean the database dropping project schemas and tables. Once the command is executed, the application would exit upon completion.', + builder: {}, + handler: (argv) => { + initLogger( + argv.debug as boolean, + argv.outputFormat as 'json' | 'colored', + argv.logLevel as string | undefined, + ); + + // lazy import to make sure logger is instantiated before all other services + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { forceCleanInit } = require('./subcommands/forceClean.init'); + return forceCleanInit(); + }, + }) + .command({ + command: 'reindex', + describe: + 'Reindex to specified block height. Historical must be enabled for the targeted project (--disable-historical=false). Once the command is executed, the application would exit upon completion.', + builder: (yargs) => + yargs.options('targetHeight', { + type: 'number', + description: 'set targetHeight', + require: true, + }), + handler: (argv) => { + initLogger( + argv.debug as boolean, + argv.outputFormat as 'json' | 'colored', + argv.logLevel as string | undefined, + ); + // lazy import to make sure logger is instantiated before all other services + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { reindexInit } = require('./subcommands/reindex.init'); + return reindexInit(argv.targetHeight); + }, + }) + .options({ subquery: { alias: 'f', demandOption: true, @@ -32,11 +72,6 @@ export function getYargsOption() { demandOption: false, describe: 'Use local mode', }, - 'force-clean': { - type: 'boolean', - demandOption: false, - describe: 'Force clean the database, dropping project schemas and tables', - }, 'db-schema': { demandOption: false, describe: 'Db schema name of the project', @@ -71,7 +106,8 @@ export function getYargsOption() { }, debug: { demandOption: false, - describe: 'Show debug information to console output. will forcefully set log level to debug', + describe: + 'Show debug information to console output. will forcefully set log level to debug', type: 'boolean', default: false, }, @@ -150,26 +186,18 @@ export function getYargsOption() { describe: 'Disable storing historical state entities', type: 'boolean', }, - reindex: { - demandOption: false, - describe: 'Reindex to specified block height', - type: 'number', - }, workers: { alias: 'w', demandOption: false, - describe: 'Number of worker threads to use for fetching and processing blocks. Disabled by default.', + describe: + 'Number of worker threads to use for fetching and processing blocks. Disabled by default.', type: 'number', }, 'query-limit': { demandOption: false, - describe: 'The limit of items a project can query with store.getByField at once', + describe: + 'The limit of items a project can query with store.getByField at once', type: 'number', default: 100, }, }); -} - -export function argv(arg: string): unknown { - return getYargsOption().argv[arg]; -} diff --git a/packages/validator/fixtures/package.json b/packages/validator/fixtures/package.json index 6a44e68e1d..ad8f0d1561 100644 --- a/packages/validator/fixtures/package.json +++ b/packages/validator/fixtures/package.json @@ -19,7 +19,7 @@ "author": "Ian He & Jay Ji", "license": "Apache-2.0", "devDependencies": { - "@subql/types": "^0.6.0", + "@subql/types": "latest", "typescript": "^4.1.3", "@subql/cli": "^0.7.3" } diff --git a/packages/validator/package.json b/packages/validator/package.json index 31b2fc4ba3..548d36c43a 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -18,7 +18,7 @@ "dependencies": { "@subql/common": "workspace:*", "@subql/common-avalanche": "latest", - "@subql/common-cosmos": "^0.0.6", + "@subql/common-cosmos": "latest", "@subql/common-substrate": "workspace:*", "@subql/common-terra": "latest", "axios": "^0.24.0", diff --git a/test/jest-setup.ts b/test/jest-setup.ts index 4e9d3815bf..fb2a138336 100644 --- a/test/jest-setup.ts +++ b/test/jest-setup.ts @@ -1,2 +1,5 @@ import 'reflect-metadata'; -import 'regenerator-runtime/runtime' +import 'regenerator-runtime/runtime'; + +// Causes the logger to be init +(global as any).__TEST__ = true; diff --git a/yarn.lock b/yarn.lock index 906937fb59..0341a59cc5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1470,6 +1470,173 @@ __metadata: languageName: node linkType: hard +"@confio/ics23@npm:^0.6.8": + version: 0.6.8 + resolution: "@confio/ics23@npm:0.6.8" + dependencies: + "@noble/hashes": ^1.0.0 + protobufjs: ^6.8.8 + checksum: 376d72f6440db60611b002b00a13e3a5bfd0d3503e7682358dbcf79641e74d8c26c234c321452fb4a758baf66eecef25d950e08bdea270486d9d03ee489e2960 + languageName: node + linkType: hard + +"@cosmjs/amino@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/amino@npm:0.28.13" + dependencies: + "@cosmjs/crypto": 0.28.13 + "@cosmjs/encoding": 0.28.13 + "@cosmjs/math": 0.28.13 + "@cosmjs/utils": 0.28.13 + checksum: 74430f975725beed9928913fb8834b6798515702bf2bf14c4ef9ec68ac1326a32959fb023c4398a48f9e0715cb03b10e7782bab38389f73aa9feb9a48f26dd67 + languageName: node + linkType: hard + +"@cosmjs/cosmwasm-stargate@npm:^0.28.9": + version: 0.28.13 + resolution: "@cosmjs/cosmwasm-stargate@npm:0.28.13" + dependencies: + "@cosmjs/amino": 0.28.13 + "@cosmjs/crypto": 0.28.13 + "@cosmjs/encoding": 0.28.13 + "@cosmjs/math": 0.28.13 + "@cosmjs/proto-signing": 0.28.13 + "@cosmjs/stargate": 0.28.13 + "@cosmjs/tendermint-rpc": 0.28.13 + "@cosmjs/utils": 0.28.13 + cosmjs-types: ^0.4.0 + long: ^4.0.0 + pako: ^2.0.2 + checksum: a7b3e95f47198d86e38026dabb53dd35a0feb23c497612b48f079c0700819be65dfa4daae8fd817a83a67482d7e180751288500ee9a0b2d131038a147e2da6d8 + languageName: node + linkType: hard + +"@cosmjs/crypto@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/crypto@npm:0.28.13" + dependencies: + "@cosmjs/encoding": 0.28.13 + "@cosmjs/math": 0.28.13 + "@cosmjs/utils": 0.28.13 + "@noble/hashes": ^1 + bn.js: ^5.2.0 + elliptic: ^6.5.3 + libsodium-wrappers: ^0.7.6 + checksum: c22d13df721399263230dff71953c0802662a395cb67645fccbdf04aa3d0b1f83736d429644f9314d74df614d63f499145a68308a27e0e3a1ae9fdcfaa933bfa + languageName: node + linkType: hard + +"@cosmjs/encoding@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/encoding@npm:0.28.13" + dependencies: + base64-js: ^1.3.0 + bech32: ^1.1.4 + readonly-date: ^1.0.0 + checksum: 681c4673edb36af3bb11cec360c3a076e6a9e56b43def6509e17e4db6d3e781c90f4e229bd664d58fbce4891f4493546178213e43a04c6a53962d34b5e00ef8b + languageName: node + linkType: hard + +"@cosmjs/json-rpc@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/json-rpc@npm:0.28.13" + dependencies: + "@cosmjs/stream": 0.28.13 + xstream: ^11.14.0 + checksum: ee12fb63b93973cdddab3977fbc77f43d6ac8c981aa23c006d6a891a0b31a218b59318ec62e2081d8eeeb6c3f7488aa669a7fe8227359a06858d611ce591d129 + languageName: node + linkType: hard + +"@cosmjs/math@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/math@npm:0.28.13" + dependencies: + bn.js: ^5.2.0 + checksum: 42b16e7d533b3985130380081ca83aab69797234d59e8851cf92ad95810e80cf7504547501020c7edad7b381a616b1b9fde298cc65bf12566d088c0f2e6f5b21 + languageName: node + linkType: hard + +"@cosmjs/proto-signing@npm:0.28.13, @cosmjs/proto-signing@npm:^0.28.9": + version: 0.28.13 + resolution: "@cosmjs/proto-signing@npm:0.28.13" + dependencies: + "@cosmjs/amino": 0.28.13 + "@cosmjs/crypto": 0.28.13 + "@cosmjs/encoding": 0.28.13 + "@cosmjs/math": 0.28.13 + "@cosmjs/utils": 0.28.13 + cosmjs-types: ^0.4.0 + long: ^4.0.0 + checksum: 3f2c41f6ec58071592cd08316f58f57714e778b704de7a8c10ea7a0400c28030e7aef72adf674ad0d8bc63746fb9a936a054d147b85e6ebd468af1a764364865 + languageName: node + linkType: hard + +"@cosmjs/socket@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/socket@npm:0.28.13" + dependencies: + "@cosmjs/stream": 0.28.13 + isomorphic-ws: ^4.0.1 + ws: ^7 + xstream: ^11.14.0 + checksum: 3396cb67b79e5f95256a4dc647c29204df42203d29a461959016c5da1a71f6835639a47ce1788853e3b9c6891ff6b0afd33209771bdd81b25fac94636f520730 + languageName: node + linkType: hard + +"@cosmjs/stargate@npm:0.28.13, @cosmjs/stargate@npm:^0.28.9": + version: 0.28.13 + resolution: "@cosmjs/stargate@npm:0.28.13" + dependencies: + "@confio/ics23": ^0.6.8 + "@cosmjs/amino": 0.28.13 + "@cosmjs/encoding": 0.28.13 + "@cosmjs/math": 0.28.13 + "@cosmjs/proto-signing": 0.28.13 + "@cosmjs/stream": 0.28.13 + "@cosmjs/tendermint-rpc": 0.28.13 + "@cosmjs/utils": 0.28.13 + cosmjs-types: ^0.4.0 + long: ^4.0.0 + protobufjs: ~6.11.3 + xstream: ^11.14.0 + checksum: 8743cfc9a68587521308420438e7f76809f82391fbfc4a5b81910691a6676d305fdb932d8c78d74fc5ab4b168d17442d3b68e098e612aaaa67a83e87569da387 + languageName: node + linkType: hard + +"@cosmjs/stream@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/stream@npm:0.28.13" + dependencies: + xstream: ^11.14.0 + checksum: ff6ff3dcd9e084a1c480af83e0d352ab545caeaa214abee7fdcc71c93462161b99165ebb903979bf91e8198403254ab7662a3dffd46fba86e92735f781a6a052 + languageName: node + linkType: hard + +"@cosmjs/tendermint-rpc@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/tendermint-rpc@npm:0.28.13" + dependencies: + "@cosmjs/crypto": 0.28.13 + "@cosmjs/encoding": 0.28.13 + "@cosmjs/json-rpc": 0.28.13 + "@cosmjs/math": 0.28.13 + "@cosmjs/socket": 0.28.13 + "@cosmjs/stream": 0.28.13 + "@cosmjs/utils": 0.28.13 + axios: ^0.21.2 + readonly-date: ^1.0.0 + xstream: ^11.14.0 + checksum: aa5f852681db43d54b971d6752cbd2ff6a2485cd24ab11abfca0f4cf725be06f9467e609fbf18a77d80ba066e5518885041f1782e6054b759753b356fe7b3d7a + languageName: node + linkType: hard + +"@cosmjs/utils@npm:0.28.13": + version: 0.28.13 + resolution: "@cosmjs/utils@npm:0.28.13" + checksum: 4db0ad8120ce9715f043985f0003c5efb1ca162a85c7a8cce7293cbd15dcf02852dee2e1ebd3a7670da943090e054ef64ed4bba7c6cd2e5c8b7cda47481c3c2c + languageName: node + linkType: hard + "@cspotcode/source-map-consumer@npm:0.8.0": version: 0.8.0 resolution: "@cspotcode/source-map-consumer@npm:0.8.0" @@ -1503,6 +1670,172 @@ __metadata: languageName: node linkType: hard +"@ethersproject/abstract-provider@npm:^5.6.1": + version: 5.7.0 + resolution: "@ethersproject/abstract-provider@npm:5.7.0" + dependencies: + "@ethersproject/bignumber": ^5.7.0 + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + "@ethersproject/networks": ^5.7.0 + "@ethersproject/properties": ^5.7.0 + "@ethersproject/transactions": ^5.7.0 + "@ethersproject/web": ^5.7.0 + checksum: 74cf4696245cf03bb7cc5b6cbf7b4b89dd9a79a1c4688126d214153a938126d4972d42c93182198653ce1de35f2a2cad68be40337d4774b3698a39b28f0228a8 + languageName: node + linkType: hard + +"@ethersproject/address@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/address@npm:5.7.0" + dependencies: + "@ethersproject/bignumber": ^5.7.0 + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/keccak256": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + "@ethersproject/rlp": ^5.7.0 + checksum: 64ea5ebea9cc0e845c413e6cb1e54e157dd9fc0dffb98e239d3a3efc8177f2ff798cd4e3206cf3660ee8faeb7bef1a47dc0ebef0d7b132c32e61e550c7d4c843 + languageName: node + linkType: hard + +"@ethersproject/base64@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/base64@npm:5.7.0" + dependencies: + "@ethersproject/bytes": ^5.7.0 + checksum: 7dd5d734d623582f08f665434f53685041a3d3b334a0e96c0c8afa8bbcaab934d50e5b6b980e826a8fde8d353e0b18f11e61faf17468177274b8e7c69cd9742b + languageName: node + linkType: hard + +"@ethersproject/bignumber@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/bignumber@npm:5.7.0" + dependencies: + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + bn.js: ^5.2.1 + checksum: 8c9a134b76f3feb4ec26a5a27379efb4e156b8fb2de0678a67788a91c7f4e30abe9d948638458e4b20f2e42380da0adacc7c9389d05fce070692edc6ae9b4904 + languageName: node + linkType: hard + +"@ethersproject/bytes@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/bytes@npm:5.7.0" + dependencies: + "@ethersproject/logger": ^5.7.0 + checksum: 66ad365ceaab5da1b23b72225c71dce472cf37737af5118181fa8ab7447d696bea15ca22e3a0e8836fdd8cfac161afe321a7c67d0dde96f9f645ddd759676621 + languageName: node + linkType: hard + +"@ethersproject/constants@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/constants@npm:5.7.0" + dependencies: + "@ethersproject/bignumber": ^5.7.0 + checksum: 6d4b1355747cce837b3e76ec3bde70e4732736f23b04f196f706ebfa5d4d9c2be50904a390d4d40ce77803b98d03d16a9b6898418e04ba63491933ce08c4ba8a + languageName: node + linkType: hard + +"@ethersproject/keccak256@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/keccak256@npm:5.7.0" + dependencies: + "@ethersproject/bytes": ^5.7.0 + js-sha3: 0.8.0 + checksum: ff70950d82203aab29ccda2553422cbac2e7a0c15c986bd20a69b13606ed8bb6e4fdd7b67b8d3b27d4f841e8222cbaccd33ed34be29f866fec7308f96ed244c6 + languageName: node + linkType: hard + +"@ethersproject/logger@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/logger@npm:5.7.0" + checksum: 075ab2f605f1fd0813f2e39c3308f77b44a67732b36e712d9bc085f22a84aac4da4f71b39bee50fe78da3e1c812673fadc41180c9970fe5e486e91ea17befe0d + languageName: node + linkType: hard + +"@ethersproject/networks@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/networks@npm:5.7.0" + dependencies: + "@ethersproject/logger": ^5.7.0 + checksum: 4f4d77e7c59e79cfcba616315a5d0e634a7653acbd11bb06a0028f4bd009b19f9a31556148a1e38f7308f55d1a1d170eb9f065290de9f9cf104b34e91cc348b8 + languageName: node + linkType: hard + +"@ethersproject/properties@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/properties@npm:5.7.0" + dependencies: + "@ethersproject/logger": ^5.7.0 + checksum: 6ab0ccf0c3aadc9221e0cdc5306ce6cd0df7f89f77d77bccdd1277182c9ead0202cd7521329ba3acde130820bf8af299e17cf567d0d497c736ee918207bbf59f + languageName: node + linkType: hard + +"@ethersproject/rlp@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/rlp@npm:5.7.0" + dependencies: + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + checksum: bce165b0f7e68e4d091c9d3cf47b247cac33252df77a095ca4281d32d5eeaaa3695d9bc06b2b057c5015353a68df89f13a4a54a72e888e4beeabbe56b15dda6e + languageName: node + linkType: hard + +"@ethersproject/signing-key@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/signing-key@npm:5.7.0" + dependencies: + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + "@ethersproject/properties": ^5.7.0 + bn.js: ^5.2.1 + elliptic: 6.5.4 + hash.js: 1.1.7 + checksum: 8f8de09b0aac709683bbb49339bc0a4cd2f95598f3546436c65d6f3c3a847ffa98e06d35e9ed2b17d8030bd2f02db9b7bd2e11c5cf8a71aad4537487ab4cf03a + languageName: node + linkType: hard + +"@ethersproject/strings@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/strings@npm:5.7.0" + dependencies: + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/constants": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + checksum: 5ff78693ae3fdf3cf23e1f6dc047a61e44c8197d2408c42719fef8cb7b7b3613a4eec88ac0ed1f9f5558c74fe0de7ae3195a29ca91a239c74b9f444d8e8b50df + languageName: node + linkType: hard + +"@ethersproject/transactions@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/transactions@npm:5.7.0" + dependencies: + "@ethersproject/address": ^5.7.0 + "@ethersproject/bignumber": ^5.7.0 + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/constants": ^5.7.0 + "@ethersproject/keccak256": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + "@ethersproject/properties": ^5.7.0 + "@ethersproject/rlp": ^5.7.0 + "@ethersproject/signing-key": ^5.7.0 + checksum: a31b71996d2b283f68486241bff0d3ea3f1ba0e8f1322a8fffc239ccc4f4a7eb2ea9994b8fd2f093283fd75f87bae68171e01b6265261f821369aca319884a79 + languageName: node + linkType: hard + +"@ethersproject/web@npm:^5.7.0": + version: 5.7.0 + resolution: "@ethersproject/web@npm:5.7.0" + dependencies: + "@ethersproject/base64": ^5.7.0 + "@ethersproject/bytes": ^5.7.0 + "@ethersproject/logger": ^5.7.0 + "@ethersproject/properties": ^5.7.0 + "@ethersproject/strings": ^5.7.0 + checksum: 9d4ca82f8b1295bbc1c59d58cb351641802d2f70f4b7d523fc726f51b0615296da6d6585dee5749b4d5e4a6a9af6d6650d46fe562d5b04f43a0af5c7f7f4a77e + languageName: node + linkType: hard + "@gar/promisify@npm:^1.0.1, @gar/promisify@npm:^1.1.3": version: 1.1.3 resolution: "@gar/promisify@npm:1.1.3" @@ -2119,7 +2452,7 @@ __metadata: languageName: node linkType: hard -"@noble/hashes@npm:1.1.2": +"@noble/hashes@npm:1.1.2, @noble/hashes@npm:^1, @noble/hashes@npm:^1.0.0": version: 1.1.2 resolution: "@noble/hashes@npm:1.1.2" checksum: 3c2a8cb7c2e053811032f242155d870c5eb98844d924d69702244d48804cb03b42d4a666c49c2b71164420d8229cb9a6f242b972d50d5bb2f1d673b98b041de2 @@ -3260,7 +3593,7 @@ __metadata: "@subql/common-avalanche": latest "@subql/common-cosmos": latest "@subql/common-substrate": "workspace:*" - "@subql/common-terra": ^0.6.0 + "@subql/common-terra": latest "@subql/utils": "workspace:*" "@subql/validator": "workspace:*" "@types/ejs": ^3.1.0 @@ -3294,8 +3627,8 @@ __metadata: linkType: soft "@subql/common-algorand@npm:latest": - version: 1.0.0 - resolution: "@subql/common-algorand@npm:1.0.0" + version: 1.0.1 + resolution: "@subql/common-algorand@npm:1.0.1" dependencies: "@subql/common": latest "@subql/types-algorand": 1.2.1 @@ -3304,17 +3637,17 @@ __metadata: js-yaml: ^4.1.0 reflect-metadata: ^0.1.13 semver: ^7.3.7 - checksum: c60bbe97468e63bb3dfa56214b15de85e205a4e4dcfae9306e936fc3e9e3072f1fa580c6ecb840d5270f3621e78f31a6621048ba79167f205c32908d6cf6e1d8 + checksum: 72c2d922472e209149902aa720715648a05cd90349a868505fea8a5f619a4a39704e94b13278efd8d2c7ca400f1f9df7d480b356deeb2db944f67011517f813b languageName: node linkType: hard "@subql/common-avalanche@npm:latest": - version: 0.0.2-0 - resolution: "@subql/common-avalanche@npm:0.0.2-0" + version: 0.2.0 + resolution: "@subql/common-avalanche@npm:0.2.0" dependencies: "@polkadot/util": ^8 - "@subql/common": 0.22.0 - "@subql/types-avalanche": 0.0.2-0 + "@subql/common": latest + "@subql/types-avalanche": 0.4.0 bn.js: 4.11.6 class-transformer: 0.4.0 class-validator: ^0.13.2 @@ -3326,35 +3659,21 @@ __metadata: reflect-metadata: ^0.1.13 sequelize: ^6.6.2 vm2: ^3.9.9 - checksum: 99493bdfb2b06af583f382584c06e583b6874740a683971d45cdd58ceff86fc0ef068659b26ac64c0205767d8ce65035aab6d588a496fdf915a3dc92c9c6646f - languageName: node - linkType: hard - -"@subql/common-cosmos@npm:^0.0.6": - version: 0.0.6 - resolution: "@subql/common-cosmos@npm:0.0.6" - dependencies: - "@subql/common": latest - "@subql/types-cosmos": 0.0.6 - class-transformer: 0.4.0 - class-validator: ^0.13.2 - js-yaml: ^4.1.0 - reflect-metadata: ^0.1.13 - checksum: 93d113560cad2202d08a51ef9e3d8e8d3dac349cb63e6ccb9744ff5ae3b728d985cec422226119219be4622e02cad1597010b0d0d5179519bc76bf1d26f4c028 + checksum: d3b18ab86f1626628c1c0a7da42c98390b559fb8c932400555357498aa4b13591da0a315b0717d398b77be6a2da394d0fa22be7d94f197b04dc8bb5a6f749fa6 languageName: node linkType: hard "@subql/common-cosmos@npm:latest": - version: 0.0.5 - resolution: "@subql/common-cosmos@npm:0.0.5" + version: 0.0.7 + resolution: "@subql/common-cosmos@npm:0.0.7" dependencies: "@subql/common": latest - "@subql/types-cosmos": 0.0.5 + "@subql/types-cosmos": 0.1.2-3 class-transformer: 0.4.0 class-validator: ^0.13.2 js-yaml: ^4.1.0 reflect-metadata: ^0.1.13 - checksum: ad0b8e64345594b991b42c9d6ae97668a4d7ac9f28ff14919c5c5b440c2b8b791d94d40a49504d8e8d5a7ca309d82c2ce8b2ecf349b8a58962bf52db341b566a + checksum: cfb0e55d2520619958ff6dc63cad7c5afab3254a00a76975216a615d78933805078dbcf0ee10c3cbb05bb83509e6fef7c9b280e0ff0b37802c0e0d7f5420cb8c languageName: node linkType: hard @@ -3374,7 +3693,7 @@ __metadata: languageName: unknown linkType: soft -"@subql/common-terra@npm:^0.6.0, @subql/common-terra@npm:latest": +"@subql/common-terra@npm:latest": version: 0.6.0 resolution: "@subql/common-terra@npm:0.6.0" dependencies: @@ -3387,20 +3706,6 @@ __metadata: languageName: node linkType: hard -"@subql/common@npm:0.22.0": - version: 0.22.0 - resolution: "@subql/common@npm:0.22.0" - dependencies: - axios: ^0.27.2 - class-transformer: 0.5.1 - class-validator: ^0.13.2 - js-yaml: ^4.1.0 - reflect-metadata: ^0.1.13 - semver: ^7.3.5 - checksum: 58259f15f19cae5f3a2aab3de8d8a6eba54c681960e32c9e83bfe7dcede185734ddd28e71518939fad35df83c9ef18b426ac7b224331b9468918ad401926f698 - languageName: node - linkType: hard - "@subql/common@npm:latest": version: 1.1.0 resolution: "@subql/common@npm:1.1.0" @@ -3565,34 +3870,25 @@ __metadata: languageName: node linkType: hard -"@subql/types-avalanche@npm:0.0.2-0": - version: 0.0.2-0 - resolution: "@subql/types-avalanche@npm:0.0.2-0" - peerDependencies: - "@polkadot/api": ^8 - checksum: 3144caabcea3049a302e515ce966e32be5cc7f68dd313418424c1d825c324d66e271d881c8efff14b6106d71d1f134e8ca63862163547ae43956a893ed228538 - languageName: node - linkType: hard - -"@subql/types-cosmos@npm:0.0.5": - version: 0.0.5 - resolution: "@subql/types-cosmos@npm:0.0.5" +"@subql/types-avalanche@npm:0.4.0": + version: 0.4.0 + resolution: "@subql/types-avalanche@npm:0.4.0" + dependencies: + "@ethersproject/abstract-provider": ^5.6.1 peerDependencies: - "@cosmjs/cosmwasm-stargate": 0.28.7 - "@cosmjs/proto-signing": 0.28.7 - "@cosmjs/stargate": 0.28.7 - checksum: 109137cd0dea1f4e63d6e94614e494d25340319c16a701aa47c60409671c918e6d713cb661812147445fab0e5a9df3cce0ae7d9ecf8feefa169e750d3d69bded + "@polkadot/api": ^9 + checksum: de8f4b13798677e6184cb015ab2698d960078f4e1572d7e8c961816bce5a7bf9da9c6d1c8d187dfb6b0ac852ecdf6c8c756d374160a0d89bb289afcf9bf6b7f2 languageName: node linkType: hard -"@subql/types-cosmos@npm:0.0.6": - version: 0.0.6 - resolution: "@subql/types-cosmos@npm:0.0.6" - peerDependencies: - "@cosmjs/cosmwasm-stargate": 0.28.7 - "@cosmjs/proto-signing": 0.28.7 - "@cosmjs/stargate": 0.28.7 - checksum: 7b86b60e0bccdbb4cd65567049a719a453036a1dc46fecfd13112d71e725e5d72395ed9ad8ce6471cb7d41859387c0f82c0c8d460a6e743c7428382518638471 +"@subql/types-cosmos@npm:0.1.2-3": + version: 0.1.2-3 + resolution: "@subql/types-cosmos@npm:0.1.2-3" + dependencies: + "@cosmjs/cosmwasm-stargate": ^0.28.9 + "@cosmjs/proto-signing": ^0.28.9 + "@cosmjs/stargate": ^0.28.9 + checksum: 74f74e49edc3016552f318cd644407a409c06eb40ccdb54dc4778e865da34d00859c2b4667b4dd0d14ca47a8878bd4eb2e5f2710b3511b29931b2c8e4f051cbe languageName: node linkType: hard @@ -3643,7 +3939,7 @@ __metadata: dependencies: "@subql/common": "workspace:*" "@subql/common-avalanche": latest - "@subql/common-cosmos": ^0.0.6 + "@subql/common-cosmos": latest "@subql/common-substrate": "workspace:*" "@subql/common-terra": latest "@types/js-yaml": ^4.0.5 @@ -5501,6 +5797,15 @@ __metadata: languageName: node linkType: hard +"axios@npm:^0.21.2": + version: 0.21.4 + resolution: "axios@npm:0.21.4" + dependencies: + follow-redirects: ^1.14.0 + checksum: 44245f24ac971e7458f3120c92f9d66d1fc695e8b97019139de5b0cc65d9b8104647db01e5f46917728edfc0cfd88eb30fc4c55e6053eef4ace76768ce95ff3c + languageName: node + linkType: hard + "axios@npm:^0.24.0": version: 0.24.0 resolution: "axios@npm:0.24.0" @@ -5656,13 +5961,20 @@ __metadata: languageName: node linkType: hard -"base64-js@npm:^1.0.2, base64-js@npm:^1.3.1": +"base64-js@npm:^1.0.2, base64-js@npm:^1.3.0, base64-js@npm:^1.3.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" checksum: 669632eb3745404c2f822a18fc3a0122d2f9a7a13f7fb8b5823ee19d1d2ff9ee5b52c53367176ea4ad093c332fd5ab4bd0ebae5a8e27917a4105a4cfc86b1005 languageName: node linkType: hard +"bech32@npm:^1.1.4": + version: 1.1.4 + resolution: "bech32@npm:1.1.4" + checksum: 0e98db619191548390d6f09ff68b0253ba7ae6a55db93dfdbb070ba234c1fd3308c0606fbcc95fad50437227b10011e2698b89f0181f6e7f845c499bd14d0f4b + languageName: node + linkType: hard + "before-after-hook@npm:^2.2.0": version: 2.2.2 resolution: "before-after-hook@npm:2.2.2" @@ -5753,6 +6065,13 @@ __metadata: languageName: node linkType: hard +"bn.js@npm:^4.11.9": + version: 4.12.0 + resolution: "bn.js@npm:4.12.0" + checksum: 39afb4f15f4ea537b55eaf1446c896af28ac948fdcf47171961475724d1bb65118cca49fa6e3d67706e4790955ec0e74de584e45c8f1ef89f46c812bee5b5a12 + languageName: node + linkType: hard + "bn.js@npm:^5.2.0": version: 5.2.0 resolution: "bn.js@npm:5.2.0" @@ -5849,6 +6168,13 @@ __metadata: languageName: node linkType: hard +"brorand@npm:^1.1.0": + version: 1.1.0 + resolution: "brorand@npm:1.1.0" + checksum: 8a05c9f3c4b46572dec6ef71012b1946db6cae8c7bb60ccd4b7dd5a84655db49fe043ecc6272e7ef1f69dc53d6730b9e2a3a03a8310509a3d797a618cbee52be + languageName: node + linkType: hard + "browser-process-hrtime@npm:^1.0.0": version: 1.0.0 resolution: "browser-process-hrtime@npm:1.0.0" @@ -6823,6 +7149,16 @@ __metadata: languageName: node linkType: hard +"cosmjs-types@npm:^0.4.0": + version: 0.4.1 + resolution: "cosmjs-types@npm:0.4.1" + dependencies: + long: ^4.0.0 + protobufjs: ~6.11.2 + checksum: 7921026bb7f1fef70a6d3c3cbfc71d6af21616d532e5cd9f2af15b94c53f98f8d76da65a8fd60f930df2a9ff4eebed1bb3f49baa3eac7117981fbc8f45259005 + languageName: node + linkType: hard + "create-require@npm:^1.1.0": version: 1.1.1 resolution: "create-require@npm:1.1.1" @@ -7329,6 +7665,21 @@ __metadata: languageName: node linkType: hard +"elliptic@npm:6.5.4, elliptic@npm:^6.5.3": + version: 6.5.4 + resolution: "elliptic@npm:6.5.4" + dependencies: + bn.js: ^4.11.9 + brorand: ^1.1.0 + hash.js: ^1.0.0 + hmac-drbg: ^1.0.1 + inherits: ^2.0.4 + minimalistic-assert: ^1.0.1 + minimalistic-crypto-utils: ^1.0.1 + checksum: d56d21fd04e97869f7ffcc92e18903b9f67f2d4637a23c860492fbbff5a3155fd9ca0184ce0c865dd6eb2487d234ce9551335c021c376cd2d3b7cb749c7d10f4 + languageName: node + linkType: hard + "emittery@npm:^0.8.1": version: 0.8.1 resolution: "emittery@npm:0.8.1" @@ -8469,6 +8820,16 @@ __metadata: languageName: node linkType: hard +"follow-redirects@npm:^1.14.0": + version: 1.15.2 + resolution: "follow-redirects@npm:1.15.2" + peerDependenciesMeta: + debug: + optional: true + checksum: faa66059b66358ba65c234c2f2a37fcec029dc22775f35d9ad6abac56003268baf41e55f9ee645957b32c7d9f62baf1f0b906e68267276f54ec4b4c597c2b190 + languageName: node + linkType: hard + "follow-redirects@npm:^1.14.4, follow-redirects@npm:^1.14.8, follow-redirects@npm:^1.14.9": version: 1.15.0 resolution: "follow-redirects@npm:1.15.0" @@ -8928,6 +9289,15 @@ __metadata: languageName: node linkType: hard +"globalthis@npm:^1.0.1": + version: 1.0.3 + resolution: "globalthis@npm:1.0.3" + dependencies: + define-properties: ^1.1.3 + checksum: fbd7d760dc464c886d0196166d92e5ffb4c84d0730846d6621a39fbbc068aeeb9c8d1421ad330e94b7bca4bb4ea092f5f21f3d36077812af5d098b4dc006c998 + languageName: node + linkType: hard + "globby@npm:^10.0.1": version: 10.0.2 resolution: "globby@npm:10.0.2" @@ -9188,6 +9558,16 @@ __metadata: languageName: node linkType: hard +"hash.js@npm:1.1.7, hash.js@npm:^1.0.0, hash.js@npm:^1.0.3": + version: 1.1.7 + resolution: "hash.js@npm:1.1.7" + dependencies: + inherits: ^2.0.3 + minimalistic-assert: ^1.0.1 + checksum: e350096e659c62422b85fa508e4b3669017311aa4c49b74f19f8e1bc7f3a54a584fdfd45326d4964d6011f2b2d882e38bea775a96046f2a61b7779a979629d8f + languageName: node + linkType: hard + "hexoid@npm:1.0.0": version: 1.0.0 resolution: "hexoid@npm:1.0.0" @@ -9202,6 +9582,17 @@ __metadata: languageName: node linkType: hard +"hmac-drbg@npm:^1.0.1": + version: 1.0.1 + resolution: "hmac-drbg@npm:1.0.1" + dependencies: + hash.js: ^1.0.3 + minimalistic-assert: ^1.0.0 + minimalistic-crypto-utils: ^1.0.1 + checksum: bd30b6a68d7f22d63f10e1888aee497d7c2c5c0bb469e66bbdac99f143904d1dfe95f8131f95b3e86c86dd239963c9d972fcbe147e7cffa00e55d18585c43fe0 + languageName: node + linkType: hard + "hoist-non-react-statics@npm:^3.3.2": version: 3.3.2 resolution: "hoist-non-react-statics@npm:3.3.2" @@ -10164,6 +10555,15 @@ __metadata: languageName: node linkType: hard +"isomorphic-ws@npm:^4.0.1": + version: 4.0.1 + resolution: "isomorphic-ws@npm:4.0.1" + peerDependencies: + ws: "*" + checksum: d7190eadefdc28bdb93d67b5f0c603385aaf87724fa2974abb382ac1ec9756ed2cfb27065cbe76122879c2d452e2982bc4314317f3d6c737ddda6c047328771a + languageName: node + linkType: hard + "istanbul-lib-coverage@npm:^3.0.0, istanbul-lib-coverage@npm:^3.2.0": version: 3.2.0 resolution: "istanbul-lib-coverage@npm:3.2.0" @@ -10827,7 +11227,7 @@ __metadata: languageName: node linkType: hard -"js-sha3@npm:^0.8.0": +"js-sha3@npm:0.8.0, js-sha3@npm:^0.8.0": version: 0.8.0 resolution: "js-sha3@npm:0.8.0" checksum: 75df77c1fc266973f06cce8309ce010e9e9f07ec35ab12022ed29b7f0d9c8757f5a73e1b35aa24840dced0dea7059085aa143d817aea9e188e2a80d569d9adce @@ -11275,6 +11675,22 @@ __metadata: languageName: node linkType: hard +"libsodium-wrappers@npm:^0.7.6": + version: 0.7.10 + resolution: "libsodium-wrappers@npm:0.7.10" + dependencies: + libsodium: ^0.7.0 + checksum: 294ac098895a15f99e65431c62478f149e9e5cbbcd1fa1b41e832b65e0ead63856cc964b3b7c14447a48701e3334661dea9223442834ae7dd0d34285991616cd + languageName: node + linkType: hard + +"libsodium@npm:^0.7.0": + version: 0.7.10 + resolution: "libsodium@npm:0.7.10" + checksum: 243794a0b3b753fafb304a82e9ff777eaccf11785bde6965e7f25171fd2fb35da302a89f009a91c1e922817d37724f7afc86592b128b2b58ed657d7fbe5259e6 + languageName: node + linkType: hard + "lilconfig@npm:2.0.4": version: 2.0.4 resolution: "lilconfig@npm:2.0.4" @@ -11849,6 +12265,20 @@ __metadata: languageName: node linkType: hard +"minimalistic-assert@npm:^1.0.0, minimalistic-assert@npm:^1.0.1": + version: 1.0.1 + resolution: "minimalistic-assert@npm:1.0.1" + checksum: cc7974a9268fbf130fb055aff76700d7e2d8be5f761fb5c60318d0ed010d839ab3661a533ad29a5d37653133385204c503bfac995aaa4236f4e847461ea32ba7 + languageName: node + linkType: hard + +"minimalistic-crypto-utils@npm:^1.0.1": + version: 1.0.1 + resolution: "minimalistic-crypto-utils@npm:1.0.1" + checksum: 6e8a0422b30039406efd4c440829ea8f988845db02a3299f372fceba56ffa94994a9c0f2fd70c17f9969eedfbd72f34b5070ead9656a34d3f71c0bd72583a0ed + languageName: node + linkType: hard + "minimatch@npm:^3.0.4, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2" @@ -12950,7 +13380,7 @@ __metadata: languageName: node linkType: hard -"pako@npm:^2.0.4": +"pako@npm:^2.0.2, pako@npm:^2.0.4": version: 2.0.4 resolution: "pako@npm:2.0.4" checksum: 82b9b0b99dd830c9103856a6dbd10f0cb2c8c32b9768184727ea381a99666de9a47a069d2e6efe6acf09336f363956b50835c196ef9311b34b7274d420eb0d88 @@ -13590,6 +14020,30 @@ __metadata: languageName: node linkType: hard +"protobufjs@npm:^6.8.8, protobufjs@npm:~6.11.2, protobufjs@npm:~6.11.3": + version: 6.11.3 + resolution: "protobufjs@npm:6.11.3" + dependencies: + "@protobufjs/aspromise": ^1.1.2 + "@protobufjs/base64": ^1.1.2 + "@protobufjs/codegen": ^2.0.4 + "@protobufjs/eventemitter": ^1.1.0 + "@protobufjs/fetch": ^1.1.0 + "@protobufjs/float": ^1.0.2 + "@protobufjs/inquire": ^1.1.0 + "@protobufjs/path": ^1.1.2 + "@protobufjs/pool": ^1.1.0 + "@protobufjs/utf8": ^1.1.0 + "@types/long": ^4.0.1 + "@types/node": ">=13.7.0" + long: ^4.0.0 + bin: + pbjs: bin/pbjs + pbts: bin/pbts + checksum: 4a6ce1964167e4c45c53fd8a312d7646415c777dd31b4ba346719947b88e61654912326101f927da387d6b6473ab52a7ea4f54d6f15d63b31130ce28e2e15070 + languageName: node + linkType: hard + "proxy-addr@npm:~2.0.7": version: 2.0.7 resolution: "proxy-addr@npm:2.0.7" @@ -13925,6 +14379,13 @@ __metadata: languageName: node linkType: hard +"readonly-date@npm:^1.0.0": + version: 1.0.0 + resolution: "readonly-date@npm:1.0.0" + checksum: 78481e2abf3c2f9bc526029458aee3e2b1c476ca1434c4cc9db5c9aba51bf8f1323c1995d764ff01f2055b01f13e05416b2e14b387f644b0a5a56554c3ee9d0a + languageName: node + linkType: hard + "receptacle@npm:^1.3.2": version: 1.3.2 resolution: "receptacle@npm:1.3.2" @@ -15319,6 +15780,13 @@ __metadata: languageName: node linkType: hard +"symbol-observable@npm:^2.0.3": + version: 2.0.3 + resolution: "symbol-observable@npm:2.0.3" + checksum: 533dcf7a7925bada60dbaa06d678e7c4966dbf0959ccba7f60c22b0494ba5d9160d6a66f2951d45a80bf20e655a89f8b91c5f0458dd12faef28716b54f91f49c + languageName: node + linkType: hard + "symbol-tree@npm:^3.2.4": version: 3.2.4 resolution: "symbol-tree@npm:3.2.4" @@ -16697,6 +17165,21 @@ __metadata: languageName: node linkType: hard +"ws@npm:^7": + version: 7.5.9 + resolution: "ws@npm:7.5.9" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: c3c100a181b731f40b7f2fddf004aa023f79d64f489706a28bc23ff88e87f6a64b3c6651fbec3a84a53960b75159574d7a7385709847a62ddb7ad6af76f49138 + languageName: node + linkType: hard + "ws@npm:^8.8.1": version: 8.8.1 resolution: "ws@npm:8.8.1" @@ -16762,6 +17245,16 @@ __metadata: languageName: node linkType: hard +"xstream@npm:^11.14.0": + version: 11.14.0 + resolution: "xstream@npm:11.14.0" + dependencies: + globalthis: ^1.0.1 + symbol-observable: ^2.0.3 + checksum: eb96b5f9cd7e6a30d18688f337b8d1c658c85bb08754f2af4247275e25c0605c8435ad8125e04ad7d606c1b760fab4679841906f92718f35f8ce327074e1375a + languageName: node + linkType: hard + "xtend@npm:^4.0.0, xtend@npm:^4.0.2, xtend@npm:~4.0.0": version: 4.0.2 resolution: "xtend@npm:4.0.2"