Subcommands for force-clean and reindex (#1281)

* draft pr for documentation * merge * force-clean command working, refactoring needed * refactor force-clean * unsafe bug, in sandboxService, export package.json bug * updated package.json node-core * subql-node starting * refactor * fix up * node test suites still broken * Fix init logger with tests * Use old logger defaults * tests passes * tidy up * builds * Fix exporting logger (#1300) * tidy up, skip db.module tests * fixed db.module * tidy up getExistingProjectSchema * Reindex command (#1307) * init reindex command * add services to module * reindex functioning * add force-clean for issue #1230 * tidy up * tidy up based on comments * remove comments * edit command des * remove reindex from nodeconfig * moved functions into node-core from utils * update version Co-authored-by: Scott Twiname <[email protected]>
subquery · Sep 22, 2022 · b0d1e85 · b0d1e85
1 parent 37bce40
commit b0d1e85
Show file tree

Hide file tree

Showing 56 changed files with 1,787 additions and 493 deletions.
diff --git a/jest.config.js b/jest.config.js
@@ -62,7 +62,7 @@ module.exports = {
   // forceCoverageMatch: [],
 
   // A path to a module which exports an async function that is triggered once before all test suites
-  // globalSetup: undefined,
+  // globalSetup: undefined
 
   // A path to a module which exports an async function that is triggered once after all test suites
   // globalTeardown: undefined,
@@ -94,6 +94,7 @@ module.exports = {
     '^@subql/common/(.*)$': '<rootDir>/packages/common/src/$1',
     '^@subql/node-core/(.*)$': '<rootDir>/packages/node-core/src/$1',
     '^@subql/utils/(.*)$': '<rootDir>/packages/utils/src/$1',
+    '^@subql/node-core/logger': '<rootDir>/packages/node-core/logger',
   },
 
   // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
@@ -140,7 +141,7 @@ module.exports = {
 
   // The paths to modules that run some code to configure or set up the testing environment before each test
   // setupFiles: [],
-  setupFiles: ['./test/jest-setup.ts'],
+  setupFiles: ['<rootDir>/test/jest-setup.ts'],
 
   // A list of paths to modules that run some code to configure or set up the testing framework before each test
   // setupFilesAfterEnv: [],

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -17,7 +17,7 @@
     "@subql/common-avalanche": "latest",
     "@subql/common-cosmos": "latest",
     "@subql/common-substrate": "workspace:*",
-    "@subql/common-terra": "^0.6.0",
+    "@subql/common-terra": "latest",
     "@subql/utils": "workspace:*",
     "@subql/validator": "workspace:*",
     "@types/ejs": "^3.1.0",

diff --git a/packages/common/fixtures/package.json b/packages/common/fixtures/package.json
@@ -9,7 +9,6 @@
     "test": "jest",
     "codegen": "./node_modules/.bin/subql codegen"
   },
-
   "homepage": "https://github.com/subquery/subql-starter",
   "repository": "github:subquery/subql-starter",
   "files": [
@@ -21,7 +20,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@polkadot/api": "^3",
-    "@subql/types": "^0.6.0",
+    "@subql/types": "latest",
     "typescript": "^4.1.3",
     "@subql/cli": "^0.7.3"
   }

diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md
@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 All logs must start with the format: [x.y.z] - yyyy-mm-dd
 
 ## [Unreleased]
+### Changes
+- Moved `yargs` file from `node-core` to `node`. (#1281)
 
 ## [0.1.1] - 2022-08-26
 ### Fixed

diff --git a/packages/node-core/logger.js b/packages/node-core/logger.js
@@ -0,0 +1,7 @@
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+// A standalone entrypoint for logger so we can import `@subql/node-core/logger`
+const logger = require('./dist/logger');
+
+module.exports = logger;
diff --git a/packages/node-core/package.json b/packages/node-core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-core",
-  "version": "0.1.2-1",
+  "version": "0.1.2-2",
   "description": "Common node features that are agnostic to blockchains",
   "homepage": "https://github.com/subquery/subql",
   "repository": "github:subquery/subql",
@@ -14,7 +14,8 @@
   "license": "Apache-2.0",
   "files": [
     "src/global.d.ts",
-    "/dist"
+    "/dist",
+    "logger.js"
   ],
   "dependencies": {
     "@nestjs/common": "^8.2.6",

diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts
@@ -33,6 +33,12 @@ export interface IConfig {
   readonly ipfs?: string;
   readonly dictionaryTimeout: number;
   readonly workers?: number;
+  readonly profiler?: boolean;
+  readonly migrate: boolean;
+  readonly unsafe?: boolean;
+  readonly subscription: boolean;
+  readonly disableHistorical: boolean;
+  readonly reindex?: number;
 }
 
 export type MinConfig = Partial<Omit<IConfig, 'subquery'>> & Pick<IConfig, 'subquery'>;
@@ -48,6 +54,10 @@ const DEFAULT_CONFIG = {
   timestampField: true,
   proofOfIndex: false,
   dictionaryTimeout: 30,
+  profiler: false,
+  migrate: false,
+  subscription: false,
+  disableHistorical: true,
 };
 
 export class NodeConfig implements IConfig {
@@ -154,6 +164,26 @@ export class NodeConfig implements IConfig {
     return this._config.workers;
   }
 
+  get profiler(): boolean {
+    return this._config.profiler;
+  }
+
+  get migrate(): boolean {
+    return this._config.migrate;
+  }
+
+  get unsafe(): boolean {
+    return this._config.unsafe;
+  }
+
+  get subscription(): boolean {
+    return this._config.subscription;
+  }
+
+  get disableHistorical(): boolean {
+    return this._config.disableHistorical;
+  }
+
   merge(config: Partial<IConfig>): this {
     assign(this._config, config);
     return this;

diff --git a/packages/node-core/src/db/db.module.test.ts b/packages/node-core/src/db/db.module.test.ts
@@ -4,9 +4,12 @@
 import {INestApplication} from '@nestjs/common';
 import {Test} from '@nestjs/testing';
 import {Sequelize} from 'sequelize';
+import {NodeConfig} from '../configure/NodeConfig';
 import {SubqueryRepo} from '../entities';
 import {DbModule} from './db.module';
 
+const nodeConfig = new NodeConfig({subquery: 'packages/node-core/test/v1.0.0', subqueryName: 'test'});
+
 describe('DbModule', () => {
   let app: INestApplication;
 
@@ -16,15 +19,7 @@ describe('DbModule', () => {
 
   it('can connect to database', async () => {
     const module = await Test.createTestingModule({
-      imports: [
-        DbModule.forRoot({
-          host: process.env.DB_HOST ?? '127.0.0.1',
-          port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432,
-          username: process.env.DB_USER ?? 'postgres',
-          password: process.env.DB_PASS ?? 'postgres',
-          database: process.env.DB_DATABASE ?? 'postgres',
-        }),
-      ],
+      imports: [DbModule.forRootWithConfig(nodeConfig)],
     }).compile();
 
     app = module.createNestApplication();
@@ -35,16 +30,7 @@ describe('DbModule', () => {
 
   it('can load subquery model', async () => {
     const module = await Test.createTestingModule({
-      imports: [
-        DbModule.forRoot({
-          host: process.env.DB_HOST ?? '127.0.0.1',
-          port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432,
-          username: process.env.DB_USER ?? 'postgres',
-          password: process.env.DB_PASS ?? 'postgres',
-          database: process.env.DB_DATABASE ?? 'postgres',
-        }),
-        DbModule.forFeature(['Subquery']),
-      ],
+      imports: [DbModule.forRootWithConfig(nodeConfig), DbModule.forFeature(['Subquery'])],
     }).compile();
 
     app = module.createNestApplication();

diff --git a/packages/node-core/src/db/db.module.ts b/packages/node-core/src/db/db.module.ts
@@ -3,10 +3,10 @@
 
 import {DynamicModule, Global} from '@nestjs/common';
 import {Sequelize, Options as SequelizeOption} from 'sequelize';
+import {NodeConfig} from '../configure/NodeConfig';
 import * as entities from '../entities';
 import {getLogger} from '../logger';
 import {delay} from '../utils/promise';
-import {getYargsOption} from '../yargs';
 
 export interface DbOption {
   host: string;
@@ -18,6 +18,14 @@ export interface DbOption {
 
 const logger = getLogger('db');
 
+const DEFAULT_DB_OPTION: DbOption = {
+  host: process.env.DB_HOST ?? '127.0.0.1',
+  port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432,
+  username: process.env.DB_USER ?? 'postgres',
+  password: process.env.DB_PASS ?? 'postgres',
+  database: process.env.DB_DATABASE ?? 'postgres',
+};
+
 async function establishConnection(sequelize: Sequelize, numRetries: number): Promise<void> {
   try {
     await sequelize.authenticate();
@@ -32,37 +40,68 @@ async function establishConnection(sequelize: Sequelize, numRetries: number): Pr
   }
 }
 
-const sequelizeFactory = (option: SequelizeOption) => async () => {
+const sequelizeFactory = (option: SequelizeOption, migrate: boolean) => async () => {
   const sequelize = new Sequelize(option);
   const numRetries = 5;
   await establishConnection(sequelize, numRetries);
   for (const factoryFn of Object.keys(entities).filter((k) => /Factory$/.exec(k))) {
     entities[factoryFn as keyof typeof entities](sequelize);
   }
-  const {migrate} = getYargsOption().argv;
   await sequelize.sync({alter: migrate});
   return sequelize;
 };
 
 @Global()
 export class DbModule {
-  static forRoot(option: DbOption): DynamicModule {
-    const {argv} = getYargsOption();
+  static forRootWithConfig(nodeConfig: NodeConfig, option: DbOption = DEFAULT_DB_OPTION): DynamicModule {
+    const logger = getLogger('db');
+
+    const factory = sequelizeFactory(
+      {
+        ...option,
+        dialect: 'postgres',
+        logging: nodeConfig.debug
+          ? (sql: string, timing?: number) => {
+              logger.debug(sql);
+            }
+          : false,
+      },
+      nodeConfig.migrate
+    )();
+
+    return {
+      module: DbModule,
+      providers: [
+        {
+          provide: Sequelize,
+          useFactory: () => factory,
+        },
+      ],
+      exports: [Sequelize],
+    };
+  }
+
+  static forRoot(option: DbOption = DEFAULT_DB_OPTION): DynamicModule {
     const logger = getLogger('db');
     return {
       module: DbModule,
       providers: [
         {
           provide: Sequelize,
-          useFactory: sequelizeFactory({
-            ...option,
-            dialect: 'postgres',
-            logging: argv.debug
-              ? (sql: string, timing?: number) => {
-                  logger.debug(sql);
-                }
-              : false,
-          }),
+          useFactory: (nodeConfig: NodeConfig) =>
+            sequelizeFactory(
+              {
+                ...option,
+                dialect: 'postgres',
+                logging: nodeConfig.debug
+                  ? (sql: string, timing?: number) => {
+                      logger.debug(sql);
+                    }
+                  : false,
+              },
+              nodeConfig.migrate
+            )(),
+          inject: [NodeConfig],
         },
       ],
       exports: [Sequelize],

diff --git a/packages/node-core/src/index.ts b/packages/node-core/src/index.ts
@@ -3,7 +3,6 @@
 
 export * from './api.service';
 export * from './logger';
-export * from './yargs';
 export * from './profiler';
 export * from './events';
 export * from './configure';

diff --git a/packages/node-core/src/indexer/benchmark.service.ts b/packages/node-core/src/indexer/benchmark.service.ts
@@ -5,14 +5,13 @@ import {OnEvent} from '@nestjs/event-emitter';
 import {Interval} from '@nestjs/schedule';
 import dayjs from 'dayjs';
 import duration from 'dayjs/plugin/duration';
+import {NodeConfig} from '../configure';
 import {IndexerEvent, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload} from '../events';
 import {getLogger} from '../logger';
 import {delay} from '../utils/promise';
-import {getYargsOption} from '../yargs';
 
 const SAMPLING_TIME_VARIANCE = 15;
 const logger = getLogger('benchmark');
-const {argv} = getYargsOption();
 dayjs.extend(duration);
 
 export class BenchmarkService {
@@ -26,6 +25,8 @@ export class BenchmarkService {
   private currentProcessedBlockAmount: number;
   private lastProcessedBlockAmount: number;
 
+  constructor(private nodeConfig: NodeConfig) {}
+
   @Interval(SAMPLING_TIME_VARIANCE * 1000)
   async benchmark(): Promise<void> {
     if (!this.currentProcessingHeight || !this.currentProcessingTimestamp || !this.currentProcessedBlockAmount) {
@@ -44,7 +45,7 @@ export class BenchmarkService {
         const days = Math.floor(blockDuration.asDays());
         const durationStr = `${days} days ${hoursMinsStr}`;
 
-        if (argv.profiler) {
+        if (this.nodeConfig.profiler) {
           logger.info(
             `Processed ${
               this.currentProcessedBlockAmount - this.lastProcessedBlockAmount

diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts
@@ -45,14 +45,12 @@ import {
   camelCaseObjectKey,
   makeTriggerName,
 } from '../utils';
-import {getYargsOption} from '../yargs';
 import {Metadata, MetadataFactory, MetadataRepo, PoiFactory, PoiRepo, ProofOfIndex} from './entities';
 import {StoreOperations} from './StoreOperations';
 import {OperationType} from './types';
 
 const logger = getLogger('store');
 const NULL_MERKEL_ROOT = hexToU8a('0x00');
-const {argv} = getYargsOption();
 const NotifyTriggerManipulationType = [`INSERT`, `DELETE`, `UPDATE`];
 const KEY_FIELDS = ['id', '__id', '__block_range'];
 
@@ -157,7 +155,7 @@ export class StoreService {
       enumTypeMap.set(e.name, `"${enumTypeName}"`);
     }
     const extraQueries = [];
-    if (argv.subscription) {
+    if (this.config.subscription) {
       extraQueries.push(createSendNotificationTriggerFunction);
     }
     for (const model of this.modelsRelations.models) {
@@ -187,7 +185,7 @@ export class StoreService {
         this.addScopeAndBlockHeightHooks(sequelizeModel);
         extraQueries.push(createExcludeConstraintQuery(schema, sequelizeModel.tableName));
       }
-      if (argv.subscription) {
+      if (this.config.subscription) {
         const triggerName = makeTriggerName(schema, sequelizeModel.tableName);
         const triggers = await this.sequelize.query(getNotifyTriggers(), {
           replacements: {triggerName},
@@ -282,7 +280,7 @@ export class StoreService {
         enabled = false;
       }
     } catch (e) {
-      enabled = !argv['disable-historical'];
+      enabled = !this.config.disableHistorical;
     }
     logger.info(`Historical state is ${enabled ? 'enabled' : 'disabled'}`);
     return enabled;