Skip to content

Commit

Permalink
Update typescript, fix most strict related errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@stwiname
stwiname committed Apr 20, 2023
1 parent 105b1f2 commit 4a89362
Show file tree
Hide file tree
Showing 38 changed files with 314 additions and 248 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"ts-loader": "^9.2.6",
"ts-node": "^10.4.0",
"tsconfig-paths": "^3.12.0",
"typescript": "^4.4.4"
"typescript": "^4.9.5"
},
"resolutions": {
"@polkadot/api": "10.1.4",
Expand Down
2 changes: 1 addition & 1 deletion packages/common/fixtures/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"devDependencies": {
"@polkadot/api": "^10",
"@subql/types": "latest",
"typescript": "^4.1.3",
"typescript": "^4.9.5",
"@subql/cli": "latest"
}
}
2 changes: 1 addition & 1 deletion packages/node-core/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"@subql/types": "workspace:*",
"@subql/utils": "workspace:*",
"@subql/x-merkle-mountain-range": "^2.0.0-0.1.2",
"@willsoto/nestjs-prometheus": "^4.4.0",
"@willsoto/nestjs-prometheus": "^5.1.1",
"async-lock": "^1.4.0",
"async-mutex": "^0.4.0",
"lodash": "^4.17.21",
Expand Down
16 changes: 6 additions & 10 deletions packages/node-core/src/api.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

import {Injectable} from '@nestjs/common';
// import {ApiWrapper} from '@subql/types-avalanche';
import {NetworkMetadataPayload} from './events';
import {ISubqueryProject} from './indexer';
import {getLogger} from './logger';

const logger = getLogger('api');
Expand All @@ -14,14 +13,11 @@ type FetchFunction<T> = (batch: number[]) => Promise<T[]>;
type FetchFunctionProvider<T> = () => FetchFunction<T>;

@Injectable()
export abstract class ApiService {
networkMeta: NetworkMetadataPayload;
export abstract class ApiService<P extends ISubqueryProject, A> {
constructor(protected project: P) {}

constructor(protected project: any) {}

abstract init(): Promise<ApiService>;
abstract get api(): any; /*ApiWrapper*/
abstract fetchBlocks(batch: number[]): Promise<any>;
abstract init(): Promise<ApiService<P, A>>;
abstract get api(): A; /*ApiWrapper*/

async fetchBlocksGeneric<T>(
fetchFuncProvider: FetchFunctionProvider<T>,
Expand All @@ -35,7 +31,7 @@ export abstract class ApiService {
// Get the latest fetch function from the provider
const fetchFunc = fetchFuncProvider();
return await fetchFunc(batch);
} catch (e) {
} catch (e: any) {
logger.error(e, `Failed to fetch blocks ${batch[0]}...${batch[batch.length - 1]}`);

reconnectAttempts++;
Expand Down
41 changes: 23 additions & 18 deletions packages/node-core/src/configure/NodeConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export interface IConfig {
readonly networkDictionary?: string;
readonly dictionaryResolver?: string;
readonly outputFmt?: 'json';
readonly logLevel?: LevelWithSilent;
readonly logLevel: LevelWithSilent;
readonly queryLimit: number;
readonly indexCountLimit: number;
readonly timestampField: boolean;
Expand All @@ -45,15 +45,16 @@ export interface IConfig {
readonly pgCa?: string;
readonly pgKey?: string;
readonly pgCert?: string;
readonly storeCacheThreshold?: number;
readonly storeGetCacheSize?: number;
readonly storeCacheAsync?: boolean;
readonly storeCacheThreshold: number;
readonly storeGetCacheSize: number;
readonly storeCacheAsync: boolean;
}

export type MinConfig = Partial<Omit<IConfig, 'subquery'>> & Pick<IConfig, 'subquery'>;

const DEFAULT_CONFIG = {
localMode: false,
logLevel: 'info',
batchSize: 100,
timeout: 900,
blockTime: 6000,
Expand Down Expand Up @@ -104,7 +105,11 @@ export class NodeConfig implements IConfig {

get subqueryName(): string {
assert(this._config.subquery);
return this._config.subqueryName ?? last(this.subquery.split(path.sep));
const name = this._config.subqueryName ?? last(this.subquery.split(path.sep));
if (!name) {
throw new Error('Unable to get subquery name');
}
return name;
}

get localMode(): boolean {
Expand Down Expand Up @@ -134,10 +139,10 @@ export class NodeConfig implements IConfig {
}

get storeCacheAsync(): boolean {
return this._config.storeCacheAsync;
return !!this._config.storeCacheAsync;
}

get dictionaryResolver(): string {
get dictionaryResolver(): string | undefined {
return this._config.dictionaryResolver;
}

Expand Down Expand Up @@ -188,24 +193,24 @@ export class NodeConfig implements IConfig {
get mmrPath(): string {
return this._config.mmrPath ?? `.mmr/${this.subqueryName}.mmr`;
}
get ipfs(): string {
get ipfs(): string | undefined {
return this._config.ipfs;
}

get dbSchema(): string {
return this._config.dbSchema ?? this.subqueryName;
}

get workers(): number {
get workers(): number | undefined {
return this._config.workers;
}

get profiler(): boolean {
return this._config.profiler;
return !!this._config.profiler;
}

get unsafe(): boolean {
return this._config.unsafe;
return !!this._config.unsafe;
}

get subscription(): boolean {
Expand All @@ -221,7 +226,7 @@ export class NodeConfig implements IConfig {
}

get unfinalizedBlocks(): boolean {
return this._config.unfinalizedBlocks;
return !!this._config.unfinalizedBlocks;
}

get isPostgresSecureConnection(): boolean {
Expand All @@ -234,8 +239,8 @@ export class NodeConfig implements IConfig {
}
try {
return getFileContent(this._config.pgCa, 'postgres ca cert');
} catch (e) {
logger.error(e);
} catch (e: any) {
logger.error(e, 'Unable to get postges CA Cert');
throw e;
}
}
Expand All @@ -247,8 +252,8 @@ export class NodeConfig implements IConfig {

try {
return getFileContent(this._config.pgKey, 'postgres client key');
} catch (e) {
logger.error(e);
} catch (e: any) {
logger.error(e, 'Unable to get postgres client key');
throw e;
}
}
Expand All @@ -259,8 +264,8 @@ export class NodeConfig implements IConfig {
}
try {
return getFileContent(this._config.pgCert, 'postgres client cert');
} catch (e) {
logger.error(e);
} catch (e: any) {
logger.error(e, 'Unable to get postgres client cert');
throw e;
}
}
Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/db/db.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ async function establishConnectionSequelize(option: SequelizeOption, numRetries:
const sequelize = new Sequelize(uri, option);
try {
await sequelize.authenticate();
} catch (error) {
} catch (error: any) {
if (JSON.stringify(error.message).includes(CONNECTION_SSL_ERROR_REGEX)) {
logger.warn('Database does not support SSL connection, will try to connect without it');
option.dialectOptions = undefined;
Expand Down
44 changes: 12 additions & 32 deletions packages/node-core/src/indexer/PoiBlock.ts
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,21 @@
// Copyright 2020-2022 OnFinality Limited authors & contributors
// SPDX-License-Identifier: Apache-2.0

import { u8aConcat, numberToU8a, hexToU8a, isHex, isU8a } from '@polkadot/util';
import { blake2AsU8a } from '@polkadot/util-crypto';
import { ProofOfIndex } from './entities/Poi.entity';
import {u8aConcat, numberToU8a, hexToU8a, isHex, isU8a} from '@polkadot/util';
import {blake2AsU8a} from '@polkadot/util-crypto';
import {ProofOfIndex} from './entities/Poi.entity';

const poiBlockHash = (
id: number,
chainBlockHash: string | Uint8Array,
operationHashRoot: Uint8Array,
parentHash: Uint8Array,
projectId: string,
projectId: string
): Uint8Array => {
if (!id || !chainBlockHash || !operationHashRoot || !projectId) {
throw new Error('Poof of index: can not generate block hash');
}
return blake2AsU8a(
u8aConcat(
numberToU8a(id),
chainBlockHash,
operationHashRoot,
Buffer.from(projectId),
parentHash,
),
);
return blake2AsU8a(u8aConcat(numberToU8a(id), chainBlockHash, operationHashRoot, Buffer.from(projectId), parentHash));
};

export class PoiBlock implements ProofOfIndex {
Expand All @@ -32,7 +24,6 @@ export class PoiBlock implements ProofOfIndex {
readonly hash: Uint8Array;
readonly parentHash: Uint8Array;
readonly operationHashRoot: Uint8Array;
mmrRoot: Uint8Array;
readonly projectId: string;

constructor(
Expand All @@ -41,7 +32,7 @@ export class PoiBlock implements ProofOfIndex {
hash: Uint8Array,
parentHash: Uint8Array,
operationHashRoot: Uint8Array,
projectId: string,
projectId: string
) {
this.id = id;
this.chainBlockHash = chainBlockHash;
Expand All @@ -56,29 +47,18 @@ export class PoiBlock implements ProofOfIndex {
chainBlockHash: string | Uint8Array,
operationHashRoot: Uint8Array,
parentHash: Uint8Array,
projectId: string,
projectId: string
): PoiBlock {
const _poiBlockHash = poiBlockHash(
id,
chainBlockHash,
operationHashRoot,
parentHash,
projectId,
);
let _chainBlockHash: Uint8Array;
const _poiBlockHash = poiBlockHash(id, chainBlockHash, operationHashRoot, parentHash, projectId);
let _chainBlockHash: Uint8Array | undefined;
if (isHex(chainBlockHash)) {
_chainBlockHash = hexToU8a(chainBlockHash);
} else if (isU8a(chainBlockHash)) {
_chainBlockHash = chainBlockHash;
} else {
throw new Error(`Unable to create PoiBlock, chainBlockHash was not valid. Received: "${chainBlockHash}"`);
}
const poiBlock = new PoiBlock(
id,
_chainBlockHash,
_poiBlockHash,
parentHash,
operationHashRoot,
projectId,
);
const poiBlock = new PoiBlock(id, _chainBlockHash, _poiBlockHash, parentHash, operationHashRoot, projectId);
return poiBlock;
}
}
64 changes: 34 additions & 30 deletions packages/node-core/src/indexer/benchmark.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ dayjs.extend(duration);

@Injectable()
export class BenchmarkService {
private currentProcessingHeight: number;
private currentProcessingTimestamp: number;
private targetHeight: number;
private lastRegisteredHeight: number;
private lastRegisteredTimestamp: number;
private blockPerSecond: number;
private currentProcessingHeight?: number;
private currentProcessingTimestamp?: number;
private targetHeight?: number;
private lastRegisteredHeight?: number;
private lastRegisteredTimestamp?: number;
private blockPerSecond?: number;

private currentProcessedBlockAmount: number;
private lastProcessedBlockAmount: number;
private currentProcessedBlockAmount?: number;
private lastProcessedBlockAmount?: number;

constructor(private nodeConfig: NodeConfig) {}

Expand All @@ -40,37 +40,41 @@ export class BenchmarkService {
const timeDiff = this.currentProcessingTimestamp - this.lastRegisteredTimestamp;
this.blockPerSecond = heightDiff === 0 || timeDiff === 0 ? 0 : heightDiff / (timeDiff / 1000);

const blockDuration = dayjs.duration(
(this.targetHeight - this.currentProcessingHeight) / this.blockPerSecond,
'seconds'
);
const hoursMinsStr = blockDuration.format('HH [hours] mm [mins]');
const days = Math.floor(blockDuration.asDays());
const durationStr = `${days} days ${hoursMinsStr}`;
if (!this.targetHeight) {
logger.debug('Target height is not defined, not logging benchmark');
} else {
const blockDuration = dayjs.duration(
(this.targetHeight - this.currentProcessingHeight) / this.blockPerSecond,
'seconds'
);
const hoursMinsStr = blockDuration.format('HH [hours] mm [mins]');
const days = Math.floor(blockDuration.asDays());
const durationStr = `${days} days ${hoursMinsStr}`;

if (this.nodeConfig.profiler && this.currentProcessedBlockAmount && this.lastProcessedBlockAmount) {
logger.info(
`Processed ${
this.currentProcessedBlockAmount - this.lastProcessedBlockAmount
} blocks in the last ${SAMPLING_TIME_VARIANCE}secs `
);
}

if (this.nodeConfig.profiler) {
logger.info(
`Processed ${
this.currentProcessedBlockAmount - this.lastProcessedBlockAmount
} blocks in the last ${SAMPLING_TIME_VARIANCE}secs `
this.targetHeight === this.lastRegisteredHeight && this.blockPerSecond === 0
? 'Fully synced, waiting for new blocks'
: `${this.blockPerSecond.toFixed(
2
)} blocks/s. Target height: ${this.targetHeight.toLocaleString()}. Current height: ${this.currentProcessingHeight.toLocaleString()}. Estimated time remaining: ${
this.blockPerSecond === 0 ? 'unknown' : durationStr
}`
);
}

logger.info(
this.targetHeight === this.lastRegisteredHeight && this.blockPerSecond === 0
? 'Fully synced, waiting for new blocks'
: `${this.blockPerSecond.toFixed(
2
)} blocks/s. Target height: ${this.targetHeight.toLocaleString()}. Current height: ${this.currentProcessingHeight.toLocaleString()}. Estimated time remaining: ${
this.blockPerSecond === 0 ? 'unknown' : durationStr
}`
);
}
this.lastRegisteredHeight = this.currentProcessingHeight;
this.lastRegisteredTimestamp = this.currentProcessingTimestamp;
this.lastProcessedBlockAmount = this.currentProcessedBlockAmount;
}
} catch (e) {
} catch (e: any) {
logger.warn(e, 'Failed to measure benchmark');
}
}
Expand Down
Loading

0 comments on commit 4a89362

Please sign in to comment.