From 85f10c7abafd65050388ab24887a9d8fc762bc83 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Thu, 18 May 2023 14:29:14 +0400 Subject: [PATCH] feat: ignore list --- docker/validators/ignore_list.yaml | 3 + src/common/config/env.validation.ts | 3 + src/duty/duty.metrics.ts | 17 +- src/duty/sync/sync.metrics.ts | 15 +- .../clickhouse/clickhouse.constants.ts | 1026 ++++++++++------- src/storage/clickhouse/clickhouse.service.ts | 143 ++- .../migration_000006_ignore_list.ts | 8 + 7 files changed, 740 insertions(+), 475 deletions(-) create mode 100644 docker/validators/ignore_list.yaml create mode 100644 src/storage/clickhouse/migrations/migration_000006_ignore_list.ts diff --git a/docker/validators/ignore_list.yaml b/docker/validators/ignore_list.yaml new file mode 100644 index 00000000..f651bc31 --- /dev/null +++ b/docker/validators/ignore_list.yaml @@ -0,0 +1,3 @@ +# this list of validators indexes is used to ignore some validators +# in metrics calculation +indexes: [] diff --git a/src/common/config/env.validation.ts b/src/common/config/env.validation.ts index f6c8077b..3d213296 100644 --- a/src/common/config/env.validation.ts +++ b/src/common/config/env.validation.ts @@ -180,6 +180,9 @@ export class EnvironmentVariables { @IsString() public VALIDATOR_REGISTRY_LIDO_SOURCE_SQLITE_CACHE_PATH = './docker/validators/lido_mainnet.db'; + @IsString() + public VALIDATOR_IGNORE_LIST = './docker/validators/ignore_list.yaml'; + /** * Distance (down) from Blockchain Sync Participation average after which we think that our sync participation is bad * For example: diff --git a/src/duty/duty.metrics.ts b/src/duty/duty.metrics.ts index 5497f7ec..89f2312d 100644 --- a/src/duty/duty.metrics.ts +++ b/src/duty/duty.metrics.ts @@ -1,8 +1,10 @@ +import { readFile } from 'fs/promises'; + import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService } from '@nestjs/common'; +import { load } from 'js-yaml'; import { ConfigService } from 'common/config'; -import { ConsensusProviderService } from 'common/eth-providers'; import { Epoch } from 'common/eth-providers/consensus-provider/types'; import { allSettled } from 'common/functions/allSettled'; import { PrometheusService, TrackTask } from 'common/prometheus'; @@ -21,7 +23,6 @@ export class DutyMetrics { @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, protected readonly config: ConfigService, protected readonly prometheus: PrometheusService, - protected readonly clClient: ConsensusProviderService, protected readonly stateMetrics: StateMetrics, protected readonly attestationMetrics: AttestationMetrics, @@ -34,6 +35,7 @@ export class DutyMetrics { @TrackTask('calc-all-duties-metrics') public async calculate(epoch: Epoch, possibleHighRewardValidators: string[]): Promise { + await this.storage.setIndexesIgnoreListForMetrics(await this.getIndexesIgnoreListForMetrics()); this.logger.log('Calculating duties metrics of user validators'); await allSettled([ this.withPossibleHighReward(epoch, possibleHighRewardValidators), @@ -52,4 +54,15 @@ export class DutyMetrics { this.syncMetrics.calculate(epoch, possibleHighRewardValidators), ]); } + + private async getIndexesIgnoreListForMetrics(): Promise { + try { + const fileContent = await readFile(this.config.get('VALIDATOR_IGNORE_LIST'), 'utf-8'); + const { indexes } = <{ indexes: string[] }>load(fileContent); + return indexes; + } catch (e) { + this.logger.error(`Error while reading ignore list file: ${e.message}`); + return []; + } + } } diff --git a/src/duty/sync/sync.metrics.ts b/src/duty/sync/sync.metrics.ts index 320d1687..ae366d37 100644 --- a/src/duty/sync/sync.metrics.ts +++ b/src/duty/sync/sync.metrics.ts @@ -71,13 +71,23 @@ export class SyncMetrics { } private async goodSyncParticipationLastEpoch(chainAvgSyncPercent: number) { - const data = await this.storage.getValidatorsCountWithGoodSyncParticipationLastNEpoch(this.processedEpoch, 1, chainAvgSyncPercent); + const data = await this.storage.getValidatorsCountWithGoodSyncParticipationLastNEpoch( + this.processedEpoch, + 1, + chainAvgSyncPercent, + this.storage.ignoreList, + ); setUserOperatorsMetric(this.prometheus.validatorsCountWithGoodSyncParticipation, data, this.operators); setOtherOperatorsMetric(this.prometheus.otherValidatorsCountWithGoodSyncParticipation, data); } private async badSyncParticipationLastEpoch(chainAvgSyncPercent: number) { - const data = await this.storage.getValidatorsCountWithBadSyncParticipationLastNEpoch(this.processedEpoch, 1, chainAvgSyncPercent); + const data = await this.storage.getValidatorsCountWithBadSyncParticipationLastNEpoch( + this.processedEpoch, + 1, + chainAvgSyncPercent, + this.storage.ignoreList, + ); setUserOperatorsMetric(this.prometheus.validatorsCountWithSyncParticipationLessAvg, data, this.operators); setOtherOperatorsMetric(this.prometheus.otherValidatorsCountWithSyncParticipationLessAvg, data); } @@ -87,6 +97,7 @@ export class SyncMetrics { this.processedEpoch, this.epochInterval, chainAvgSyncPercent, + this.storage.ignoreList, ); setUserOperatorsMetric(this.prometheus.validatorsCountWithSyncParticipationLessAvgLastNEpoch, data, this.operators, { epoch_interval: this.epochInterval, diff --git a/src/storage/clickhouse/clickhouse.constants.ts b/src/storage/clickhouse/clickhouse.constants.ts index 60645d09..f4c779ca 100644 --- a/src/storage/clickhouse/clickhouse.constants.ts +++ b/src/storage/clickhouse/clickhouse.constants.ts @@ -1,145 +1,180 @@ import { ValStatus } from 'common/eth-providers'; import { Epoch } from 'common/eth-providers/consensus-provider/types'; -export const avgValidatorBalanceDelta = (epoch: Epoch): string => ` - SELECT - current.val_nos_id as val_nos_id, - avg(current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) AS amount - FROM ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_status != '${ValStatus.PendingQueued}' AND - val_nos_id IS NOT NULL AND - epoch = ${epoch} - LIMIT 1 BY val_id - ) AS current - INNER JOIN ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_status != '${ValStatus.PendingQueued}' AND - val_nos_id IS NOT NULL AND - epoch = (${epoch} - 6) - LIMIT 1 BY val_id - ) AS previous - ON - previous.val_id = current.val_id - LEFT JOIN ( +export const avgValidatorBalanceDelta = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + current.val_nos_id as val_nos_id, + avg(current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) AS amount FROM ( - SELECT val_balance_withdrawn, val_id, val_nos_id + SELECT val_balance, val_id, val_nos_id FROM validators_summary WHERE + val_status != '${ValStatus.PendingQueued}' AND val_nos_id IS NOT NULL AND - val_balance_withdrawn > 0 AND - epoch > (${epoch} - 6) AND epoch <= ${epoch} - LIMIT 1 BY epoch, val_id - ) - GROUP BY val_id, val_nos_id - ) AS withdrawals - ON - withdrawals.val_id = current.val_id - GROUP BY current.val_nos_id -`; + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS current + INNER JOIN ( + SELECT val_balance, val_id, val_nos_id + FROM validators_summary + WHERE + val_status != '${ValStatus.PendingQueued}' AND + val_nos_id IS NOT NULL AND + epoch = (${epoch} - 6) + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS previous + ON + previous.val_id = current.val_id + LEFT JOIN ( + SELECT + sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + FROM ( + SELECT val_balance_withdrawn, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + val_balance_withdrawn > 0 AND + epoch > (${epoch} - 6) AND epoch <= ${epoch} + ${strIgnoreList} + LIMIT 1 BY epoch, val_id + ) + GROUP BY val_id, val_nos_id + ) AS withdrawals + ON + withdrawals.val_id = current.val_id + GROUP BY current.val_nos_id + `; +}; -export const validatorQuantile0001BalanceDeltasQuery = (epoch: Epoch): string => ` - SELECT - current.val_nos_id as val_nos_id, - quantileExact(0.001)(current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) AS amount - FROM ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_status != '${ValStatus.PendingQueued}' AND - val_nos_id IS NOT NULL AND - epoch = ${epoch} - LIMIT 1 BY val_id - ) AS current - INNER JOIN ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_status != '${ValStatus.PendingQueued}' AND - val_nos_id IS NOT NULL AND - epoch = (${epoch} - 6) - LIMIT 1 BY val_id - ) AS previous - ON - previous.val_id = current.val_id - LEFT JOIN ( +export const validatorQuantile0001BalanceDeltasQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + current.val_nos_id as val_nos_id, + quantileExact(0.001)(current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) AS amount FROM ( - SELECT val_balance_withdrawn, val_id, val_nos_id + SELECT val_balance, val_id, val_nos_id FROM validators_summary WHERE + val_status != '${ValStatus.PendingQueued}' AND val_nos_id IS NOT NULL AND - val_balance_withdrawn > 0 AND - epoch > (${epoch} - 6) AND epoch <= ${epoch} - LIMIT 1 BY epoch, val_id - ) - GROUP BY val_id, val_nos_id - ) AS withdrawals - ON - withdrawals.val_id = current.val_id - GROUP BY current.val_nos_id -`; - -export const validatorsCountWithNegativeDeltaQuery = (epoch: Epoch): string => ` - SELECT - current.val_nos_id as val_nos_id, - count(current.val_id) AS amount - FROM ( - SELECT val_balance, val_id, val_nos_id, val_slashed + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS current + INNER JOIN ( + SELECT val_balance, val_id, val_nos_id FROM validators_summary WHERE val_status != '${ValStatus.PendingQueued}' AND val_nos_id IS NOT NULL AND - epoch = ${epoch} + epoch = (${epoch} - 6) + ${strIgnoreList} LIMIT 1 BY val_id - ) AS current - INNER JOIN ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_status != '${ValStatus.PendingQueued}' AND - val_nos_id IS NOT NULL AND - epoch = (${epoch} - 6) - LIMIT 1 BY val_id - ) AS previous - ON - previous.val_id = current.val_id - LEFT JOIN ( + ) AS previous + ON + previous.val_id = current.val_id + LEFT JOIN ( + SELECT + sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + FROM ( + SELECT val_balance_withdrawn, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + val_balance_withdrawn > 0 AND + epoch > (${epoch} - 6) AND epoch <= ${epoch} + ${strIgnoreList} + LIMIT 1 BY epoch, val_id + ) + GROUP BY val_id, val_nos_id + ) AS withdrawals + ON + withdrawals.val_id = current.val_id + GROUP BY current.val_nos_id + `; +}; + +export const validatorsCountWithNegativeDeltaQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + current.val_nos_id as val_nos_id, + count(current.val_id) AS amount FROM ( - SELECT val_balance_withdrawn, val_id, val_nos_id + SELECT val_balance, val_id, val_nos_id, val_slashed + FROM validators_summary + WHERE + val_status != '${ValStatus.PendingQueued}' AND + val_nos_id IS NOT NULL AND + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS current + INNER JOIN ( + SELECT val_balance, val_id, val_nos_id FROM validators_summary WHERE + val_status != '${ValStatus.PendingQueued}' AND val_nos_id IS NOT NULL AND - val_balance_withdrawn > 0 AND - epoch > (${epoch} - 6) AND epoch <= ${epoch} - LIMIT 1 BY epoch, val_id - ) - GROUP BY val_id, val_nos_id - ) AS withdrawals - ON - withdrawals.val_id = current.val_id - GROUP BY current.val_nos_id - HAVING (current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) < 0 AND current.val_slashed = 0 -`; + epoch = (${epoch} - 6) + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS previous + ON + previous.val_id = current.val_id + LEFT JOIN ( + SELECT + sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + FROM ( + SELECT val_balance_withdrawn, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + val_balance_withdrawn > 0 AND + epoch > (${epoch} - 6) AND epoch <= ${epoch} + ${strIgnoreList} + LIMIT 1 BY epoch, val_id + ) + GROUP BY val_id, val_nos_id + ) AS withdrawals + ON + withdrawals.val_id = current.val_id + GROUP BY current.val_nos_id + HAVING (current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) < 0 AND current.val_slashed = 0 + `; +}; export const validatorsCountWithSyncParticipationByConditionLastNEpochQuery = ( epoch: Epoch, epochInterval: number, - validatorIndexes: string[] = [], + ignoreList: string[] = [], + neededList: string[] = [], condition: string, ): string => { - let strFilterValIndexes = ''; - if (validatorIndexes.length > 0) { - strFilterValIndexes = `AND val_id in [${validatorIndexes.map((i) => `'${i}'`).join(',')}]`; + let strNeededList = ''; + let strIgnoreList = ''; + if (neededList.length > 0) { + strNeededList = `AND val_id IN [${neededList + .filter((v) => !ignoreList.includes(v)) + .map((i) => `'${i}'`) + .join(',')}]`; + } + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; } return ` SELECT @@ -156,7 +191,8 @@ export const validatorsCountWithSyncParticipationByConditionLastNEpochQuery = ( is_sync = 1 AND ${condition} AND (epoch <= ${epoch} AND epoch > (${epoch} - ${epochInterval})) - ${strFilterValIndexes} + ${strNeededList} + ${strIgnoreList} LIMIT 1 BY epoch, val_id ) GROUP BY val_id, val_nos_id @@ -169,12 +205,20 @@ export const validatorsCountWithSyncParticipationByConditionLastNEpochQuery = ( export const validatorCountByConditionAttestationLastNEpochQuery = ( epoch: Epoch, epochInterval: number, - validatorIndexes: string[] = [], + ignoreList: string[] = [], + neededList: string[] = [], condition: string, ): string => { - let strFilterValIndexes = ''; - if (validatorIndexes.length > 0) { - strFilterValIndexes = `AND val_id in [${validatorIndexes.map((i) => `'${i}'`).join(',')}]`; + let strNeededList = ''; + let strIgnoreList = ''; + if (neededList.length > 0) { + strNeededList = `AND val_id IN [${neededList + .filter((v) => !ignoreList.includes(v)) + .map((i) => `'${i}'`) + .join(',')}]`; + } + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; } return ` SELECT @@ -190,7 +234,8 @@ export const validatorCountByConditionAttestationLastNEpochQuery = ( WHERE ${condition} AND (epoch <= ${epoch} AND epoch > (${epoch} - ${epochInterval})) - ${strFilterValIndexes} + ${strIgnoreList} + ${strNeededList} LIMIT 1 BY epoch, val_id ) GROUP BY val_id, val_nos_id @@ -200,7 +245,15 @@ export const validatorCountByConditionAttestationLastNEpochQuery = ( `; }; -export const validatorCountHighAvgIncDelayAttestationOfNEpochQuery = (epoch: Epoch, epochInterval: number): string => { +export const validatorCountHighAvgIncDelayAttestationOfNEpochQuery = ( + epoch: Epoch, + epochInterval: number, + ignoreList: string[], +): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } return ` SELECT val_nos_id, @@ -214,6 +267,7 @@ export const validatorCountHighAvgIncDelayAttestationOfNEpochQuery = (epoch: Epo FROM validators_summary WHERE (epoch <= ${epoch} AND epoch > (${epoch} - ${epochInterval})) + ${strIgnoreList} LIMIT 1 BY epoch, val_id ) GROUP BY val_id, val_nos_id @@ -223,10 +277,22 @@ export const validatorCountHighAvgIncDelayAttestationOfNEpochQuery = (epoch: Epo `; }; -export const validatorsCountByConditionMissProposeQuery = (epoch: Epoch, validatorIndexes: string[] = [], condition: string): string => { - let strFilterValIndexes = ''; - if (validatorIndexes.length > 0) { - strFilterValIndexes = `AND val_id in [${validatorIndexes.map((i) => `'${i}'`).join(',')}]`; +export const validatorsCountByConditionMissProposeQuery = ( + epoch: Epoch, + ignoreList: string[], + neededList: string[] = [], + condition: string, +): string => { + let strNeededList = ''; + let strIgnoreList = ''; + if (neededList.length > 0) { + strNeededList = `AND val_id IN [${neededList + .filter((v) => !ignoreList.includes(v)) + .map((i) => `'${i}'`) + .join(',')}]`; + } + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; } return ` SELECT @@ -239,24 +305,34 @@ export const validatorsCountByConditionMissProposeQuery = (epoch: Epoch, validat is_proposer = 1 AND ${condition} AND (epoch <= ${epoch} AND epoch > (${epoch} - 1)) - ${strFilterValIndexes} + ${strNeededList} + ${strIgnoreList} LIMIT 1 BY epoch, val_id ) GROUP BY val_nos_id `; }; -export const userSyncParticipationAvgPercentQuery = (epoch: Epoch): string => ` - SELECT - avg(sync_percent) as amount - FROM ( - SELECT sync_percent - FROM validators_summary - WHERE - is_sync = 1 AND val_nos_id IS NOT NULL AND epoch = ${epoch} - LIMIT 1 BY val_id - ) -`; +export const userSyncParticipationAvgPercentQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` + SELECT + avg(sync_percent) as amount + FROM ( + SELECT sync_percent + FROM validators_summary + WHERE + is_sync = 1 AND + val_nos_id IS NOT NULL AND + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) + `; +}; export const otherSyncParticipationAvgPercentQuery = (epoch: Epoch): string => ` SELECT @@ -282,180 +358,223 @@ export const chainSyncParticipationAvgPercentQuery = (epoch: Epoch): string => ` ) `; -export const operatorsSyncParticipationAvgPercentsQuery = (epoch: Epoch): string => ` - SELECT - val_nos_id, - avg(sync_percent) as amount - FROM ( - SELECT val_nos_id, sync_percent - FROM validators_summary - WHERE - is_sync = 1 AND val_nos_id IS NOT NULL AND epoch = ${epoch} - LIMIT 1 BY val_id - ) - GROUP BY val_nos_id -`; - -export const totalBalance24hDifferenceQuery = (epoch: Epoch): string => ` - SELECT - SUM(curr.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) as amount - FROM ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - epoch = ${epoch} - AND val_status != '${ValStatus.PendingQueued}' - AND val_nos_id IS NOT NULL - LIMIT 1 BY val_id - ) as curr - INNER JOIN ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_status != '${ValStatus.PendingQueued}' AND - val_nos_id IS NOT NULL AND - epoch = ${epoch} - 225 - LIMIT 1 BY val_id - ) AS previous - ON - previous.val_nos_id = curr.val_nos_id AND - previous.val_id = curr.val_id - LEFT JOIN ( +export const operatorsSyncParticipationAvgPercentsQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + val_nos_id, + avg(sync_percent) as amount FROM ( - SELECT val_balance_withdrawn, val_id, val_nos_id + SELECT val_nos_id, sync_percent FROM validators_summary WHERE + is_sync = 1 AND val_nos_id IS NOT NULL AND - val_balance_withdrawn > 0 AND - epoch > (${epoch} - 225) AND epoch <= ${epoch} - LIMIT 1 BY epoch, val_id + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id ) - GROUP BY val_id, val_nos_id - ) AS withdrawals - ON - withdrawals.val_nos_id = curr.val_nos_id AND - withdrawals.val_id = curr.val_id -`; + GROUP BY val_nos_id + `; +}; -export const operatorBalance24hDifferenceQuery = (epoch: Epoch): string => ` - SELECT - curr.val_nos_id as val_nos_id, - SUM(curr.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) as amount - FROM ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - epoch = ${epoch} - AND val_status != '${ValStatus.PendingQueued}' - AND val_nos_id IS NOT NULL - LIMIT 1 BY val_id - ) as curr - INNER JOIN ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_status != '${ValStatus.PendingQueued}' AND - val_nos_id IS NOT NULL AND - epoch = ${epoch} - 225 - LIMIT 1 BY val_id - ) AS previous - ON - previous.val_nos_id = curr.val_nos_id AND - previous.val_id = curr.val_id - LEFT JOIN ( +export const totalBalance24hDifferenceQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + SUM(curr.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) as amount FROM ( - SELECT val_balance_withdrawn, val_id, val_nos_id + SELECT val_balance, val_id, val_nos_id FROM validators_summary WHERE + epoch = ${epoch} + AND val_status != '${ValStatus.PendingQueued}' + AND val_nos_id IS NOT NULL + ${strIgnoreList} + LIMIT 1 BY val_id + ) as curr + INNER JOIN ( + SELECT val_balance, val_id, val_nos_id + FROM validators_summary + WHERE + val_status != '${ValStatus.PendingQueued}' AND val_nos_id IS NOT NULL AND - val_balance_withdrawn > 0 AND - epoch > (${epoch} - 225) AND epoch <= ${epoch} - LIMIT 1 BY epoch, val_id - ) - GROUP BY val_id, val_nos_id - ) AS withdrawals - ON - withdrawals.val_nos_id = curr.val_nos_id AND - withdrawals.val_id = curr.val_id - GROUP BY curr.val_nos_id -`; + epoch = ${epoch} - 225 + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS previous + ON + previous.val_nos_id = curr.val_nos_id AND + previous.val_id = curr.val_id + LEFT JOIN ( + SELECT + sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + FROM ( + SELECT val_balance_withdrawn, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + val_balance_withdrawn > 0 AND + epoch > (${epoch} - 225) AND epoch <= ${epoch} + ${strIgnoreList} + LIMIT 1 BY epoch, val_id + ) + GROUP BY val_id, val_nos_id + ) AS withdrawals + ON + withdrawals.val_nos_id = curr.val_nos_id AND + withdrawals.val_id = curr.val_id + `; +}; -export const userNodeOperatorsStatsQuery = (epoch: Epoch): string => ` - SELECT - val_nos_id, - SUM(a) as active_ongoing, - SUM(p) as pending, - SUM(s) as slashed, - ifNull(SUM(wp), 0) as withdraw_pending, - ifNull(SUM(w), 0) as withdrawn - FROM ( +export const operatorBalance24hDifferenceQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - val_nos_id, - IF(val_status = '${ValStatus.ActiveOngoing}', count(val_status), 0) as a, - IF(val_status = '${ValStatus.PendingQueued}' OR val_status = '${ValStatus.PendingInitialized}', count(val_status), 0) as p, - IF(val_status = '${ValStatus.ActiveSlashed}' OR val_status = '${ValStatus.ExitedSlashed}' OR val_slashed = 1, count(val_status), 0) as s, - IF( - (val_status in ['${ValStatus.ActiveExiting}','${ValStatus.ExitedUnslashed}', '${ValStatus.ExitedSlashed}']) - OR - (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance != 0), - count(val_status), 0 - ) as wp, - IF( - (val_status == '${ValStatus.WithdrawalDone}') - OR - (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance == 0), - count(val_status), 0 - ) as w + curr.val_nos_id as val_nos_id, + SUM(curr.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) as amount FROM ( - SELECT val_nos_id, val_status, val_slashed, val_balance + SELECT val_balance, val_id, val_nos_id FROM validators_summary WHERE - val_nos_id IS NOT NULL and epoch = ${epoch} + epoch = ${epoch} + AND val_status != '${ValStatus.PendingQueued}' + AND val_nos_id IS NOT NULL + ${strIgnoreList} + LIMIT 1 BY val_id + ) as curr + INNER JOIN ( + SELECT val_balance, val_id, val_nos_id + FROM validators_summary + WHERE + val_status != '${ValStatus.PendingQueued}' AND + val_nos_id IS NOT NULL AND + epoch = ${epoch} - 225 + ${strIgnoreList} LIMIT 1 BY val_id + ) AS previous + ON + previous.val_nos_id = curr.val_nos_id AND + previous.val_id = curr.val_id + LEFT JOIN ( + SELECT + sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + FROM ( + SELECT val_balance_withdrawn, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + val_balance_withdrawn > 0 AND + epoch > (${epoch} - 225) AND epoch <= ${epoch} + ${strIgnoreList} + LIMIT 1 BY epoch, val_id + ) + GROUP BY val_id, val_nos_id + ) AS withdrawals + ON + withdrawals.val_nos_id = curr.val_nos_id AND + withdrawals.val_id = curr.val_id + GROUP BY curr.val_nos_id + `; +}; + +export const userNodeOperatorsStatsQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` + SELECT + val_nos_id, + SUM(a) as active_ongoing, + SUM(p) as pending, + SUM(s) as slashed, + ifNull(SUM(wp), 0) as withdraw_pending, + ifNull(SUM(w), 0) as withdrawn + FROM ( + SELECT + val_nos_id, + IF(val_status = '${ValStatus.ActiveOngoing}', count(val_status), 0) as a, + IF(val_status = '${ValStatus.PendingQueued}' OR val_status = '${ValStatus.PendingInitialized}', count(val_status), 0) as p, + IF(val_status = '${ValStatus.ActiveSlashed}' OR val_status = '${ValStatus.ExitedSlashed}' OR val_slashed = 1, count(val_status), 0) as s, + IF( + (val_status in ['${ValStatus.ActiveExiting}','${ValStatus.ExitedUnslashed}', '${ValStatus.ExitedSlashed}']) + OR + (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance != 0), + count(val_status), 0 + ) as wp, + IF( + (val_status == '${ValStatus.WithdrawalDone}') + OR + (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance == 0), + count(val_status), 0 + ) as w + FROM ( + SELECT val_nos_id, val_status, val_slashed, val_balance + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) + GROUP BY val_nos_id, val_status, val_slashed, val_balance ) - GROUP BY val_nos_id, val_status, val_slashed, val_balance - ) - GROUP by val_nos_id -`; + GROUP by val_nos_id + `; +}; -export const userValidatorsSummaryStatsQuery = (epoch: Epoch): string => ` - SELECT - SUM(a) as active_ongoing, - SUM(p) as pending, - SUM(s) as slashed, - ifNull(SUM(wp), 0) as withdraw_pending, - ifNull(SUM(w), 0) as withdrawn - FROM ( +export const userValidatorsSummaryStatsQuery = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - IF(val_status = '${ValStatus.ActiveOngoing}', count(val_status), 0) as a, - IF(val_status = '${ValStatus.PendingQueued}' OR val_status = '${ValStatus.PendingInitialized}', count(val_status), 0) as p, - IF(val_status = '${ValStatus.ActiveSlashed}' OR val_status = '${ValStatus.ExitedSlashed}' OR val_slashed = 1, count(val_status), 0) as s, - IF( - (val_status in ['${ValStatus.ActiveExiting}','${ValStatus.ExitedUnslashed}', '${ValStatus.ExitedSlashed}']) - OR - (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance != 0), - count(val_status), 0 - ) as wp, - IF( - (val_status == '${ValStatus.WithdrawalDone}') - OR - (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance == 0), - count(val_status), 0 - ) as w + SUM(a) as active_ongoing, + SUM(p) as pending, + SUM(s) as slashed, + ifNull(SUM(wp), 0) as withdraw_pending, + ifNull(SUM(w), 0) as withdrawn FROM ( - SELECT val_status, val_slashed, val_balance - FROM validators_summary - WHERE - val_nos_id IS NOT NULL and epoch = ${epoch} - LIMIT 1 BY val_id + SELECT + IF(val_status = '${ValStatus.ActiveOngoing}', count(val_status), 0) as a, + IF(val_status = '${ValStatus.PendingQueued}' OR val_status = '${ValStatus.PendingInitialized}', count(val_status), 0) as p, + IF(val_status = '${ValStatus.ActiveSlashed}' OR val_status = '${ValStatus.ExitedSlashed}' OR val_slashed = 1, count(val_status), 0) as s, + IF( + (val_status in ['${ValStatus.ActiveExiting}','${ValStatus.ExitedUnslashed}', '${ValStatus.ExitedSlashed}']) + OR + (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance != 0), + count(val_status), 0 + ) as wp, + IF( + (val_status == '${ValStatus.WithdrawalDone}') + OR + (val_status == '${ValStatus.WithdrawalPossible}' AND val_balance == 0), + count(val_status), 0 + ) as w + FROM ( + SELECT val_status, val_slashed, val_balance + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) + GROUP BY val_status, val_slashed, val_balance ) - GROUP BY val_status, val_slashed, val_balance - ) -`; + `; +}; export const otherValidatorsSummaryStatsQuery = (epoch: Epoch): string => ` SELECT @@ -492,27 +611,35 @@ export const otherValidatorsSummaryStatsQuery = (epoch: Epoch): string => ` ) `; -export const userNodeOperatorsProposesStatsLastNEpochQuery = (epoch: Epoch, epochInterval = 120): string => ` - SELECT - val_nos_id, - SUM(a) as all, - SUM(m) as missed - FROM ( +export const userNodeOperatorsProposesStatsLastNEpochQuery = (epoch: Epoch, epochInterval = 120, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT val_nos_id, - count(block_proposed) as a, - IF(block_proposed = 0, count(block_proposed), 0) as m + SUM(a) as all, + SUM(m) as missed FROM ( - SELECT val_nos_id, block_proposed - FROM validators_summary - WHERE - is_proposer = 1 AND (epoch <= ${epoch} AND epoch > (${epoch} - ${epochInterval})) - LIMIT 1 BY epoch, val_id + SELECT + val_nos_id, + count(block_proposed) as a, + IF(block_proposed = 0, count(block_proposed), 0) as m + FROM ( + SELECT val_nos_id, block_proposed + FROM validators_summary + WHERE + is_proposer = 1 AND + (epoch <= ${epoch} AND epoch > (${epoch} - ${epochInterval})) + ${strIgnoreList} + LIMIT 1 BY epoch, val_id + ) + GROUP BY val_nos_id, block_proposed ) - GROUP BY val_nos_id, block_proposed - ) - GROUP by val_nos_id -`; + GROUP by val_nos_id + `; +}; export const epochMetadata = (epoch: Epoch): string => ` SELECT * @@ -526,107 +653,127 @@ export const epochProcessing = (epoch: Epoch): string => ` WHERE epoch = ${epoch} `; -export const userNodeOperatorsRewardsAndPenaltiesStats = (epoch: Epoch): string => ` - SELECT - att.val_nos_id as val_nos_id, - -- - attestation_reward as att_reward, - ifNull(prop_reward, 0) as prop_reward, - ifNull(sync_reward, 0) as sync_reward, - attestation_missed as att_missed, - ifNull(prop_missed, 0) as prop_missed, - ifNull(sync_missed, 0) as sync_missed, - attestation_penalty as att_penalty, - ifNull(prop_penalty, 0) as prop_penalty, - ifNull(sync_penalty, 0) as sync_penalty, - -- - att_reward + prop_reward + sync_reward as total_reward, - att_missed + prop_missed + sync_missed as total_missed, - att_penalty + prop_penalty + sync_penalty as total_penalty, - total_reward - total_penalty as calculated_balance_change, - real_balance_change, - calculated_balance_change - real_balance_change as calculation_error - FROM ( - SELECT - val_nos_id, - sum(att_earned_reward) as attestation_reward, - sum(att_missed_reward) as attestation_missed, - sum(att_penalty) as attestation_penalty - FROM ( - SELECT val_nos_id, att_earned_reward, att_missed_reward, att_penalty - FROM validators_summary - WHERE val_nos_id IS NOT NULL and epoch = ${epoch} - 1 - LIMIT 1 BY val_id - ) - GROUP BY val_nos_id - ) as att - LEFT JOIN ( - SELECT - val_nos_id, - sum(propose_earned_reward) as prop_reward, - sum(propose_missed_reward) as prop_missed, - sum(propose_penalty) as prop_penalty - FROM ( - SELECT val_nos_id, propose_earned_reward, propose_missed_reward, propose_penalty - FROM validators_summary - WHERE val_nos_id IS NOT NULL and epoch = ${epoch} and is_proposer = 1 - LIMIT 1 BY val_id - ) - GROUP BY val_nos_id - ) as prop ON att.val_nos_id = prop.val_nos_id - LEFT JOIN ( - SELECT - val_nos_id, - sum(sync_earned_reward) as sync_reward, - sum(sync_missed_reward) as sync_missed, - sum(sync_penalty) as sync_penalty - FROM ( - SELECT val_nos_id, sync_earned_reward, sync_missed_reward, sync_penalty - FROM validators_summary - WHERE val_nos_id IS NOT NULL and epoch = ${epoch} and is_sync = 1 - LIMIT 1 BY val_id - ) - GROUP BY val_nos_id - ) as sync ON att.val_nos_id = sync.val_nos_id - LEFT JOIN ( +export const userNodeOperatorsRewardsAndPenaltiesStats = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` SELECT - current.val_nos_id as val_nos_id, - sum(current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) AS real_balance_change + att.val_nos_id as val_nos_id, + -- + attestation_reward as att_reward, + ifNull(prop_reward, 0) as prop_reward, + ifNull(sync_reward, 0) as sync_reward, + attestation_missed as att_missed, + ifNull(prop_missed, 0) as prop_missed, + ifNull(sync_missed, 0) as sync_missed, + attestation_penalty as att_penalty, + ifNull(prop_penalty, 0) as prop_penalty, + ifNull(sync_penalty, 0) as sync_penalty, + -- + att_reward + prop_reward + sync_reward as total_reward, + att_missed + prop_missed + sync_missed as total_missed, + att_penalty + prop_penalty + sync_penalty as total_penalty, + total_reward - total_penalty as calculated_balance_change, + real_balance_change, + calculated_balance_change - real_balance_change as calculation_error FROM ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary as curr - WHERE - val_nos_id IS NOT NULL AND - epoch = ${epoch} - LIMIT 1 BY val_id - ) AS current - INNER JOIN ( - SELECT val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_nos_id IS NOT NULL AND - epoch = (${epoch} - 1) - LIMIT 1 BY val_id - ) AS previous ON previous.val_id = current.val_id + SELECT + val_nos_id, + sum(att_earned_reward) as attestation_reward, + sum(att_missed_reward) as attestation_missed, + sum(att_penalty) as attestation_penalty + FROM ( + SELECT val_nos_id, att_earned_reward, att_missed_reward, att_penalty + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + epoch = ${epoch} - 1 + ${strIgnoreList} + LIMIT 1 BY val_id + ) + GROUP BY val_nos_id + ) as att LEFT JOIN ( SELECT - sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + val_nos_id, + sum(propose_earned_reward) as prop_reward, + sum(propose_missed_reward) as prop_missed, + sum(propose_penalty) as prop_penalty FROM ( - SELECT val_balance_withdrawn, val_id, val_nos_id + SELECT val_nos_id, propose_earned_reward, propose_missed_reward, propose_penalty FROM validators_summary WHERE val_nos_id IS NOT NULL AND - val_balance_withdrawn > 0 AND - epoch = ${epoch} + epoch = ${epoch} AND + is_proposer = 1 + ${strIgnoreList} LIMIT 1 BY val_id ) - GROUP BY val_id, val_nos_id - ) AS withdrawals - ON - withdrawals.val_id = current.val_id - GROUP BY current.val_nos_id - ) as bal ON att.val_nos_id = bal.val_nos_id -`; + GROUP BY val_nos_id + ) as prop ON att.val_nos_id = prop.val_nos_id + LEFT JOIN ( + SELECT + val_nos_id, + sum(sync_earned_reward) as sync_reward, + sum(sync_missed_reward) as sync_missed, + sum(sync_penalty) as sync_penalty + FROM ( + SELECT val_nos_id, sync_earned_reward, sync_missed_reward, sync_penalty + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + epoch = ${epoch} AND + is_sync = 1 + ${strIgnoreList} + LIMIT 1 BY val_id + ) + GROUP BY val_nos_id + ) as sync ON att.val_nos_id = sync.val_nos_id + LEFT JOIN ( + SELECT + current.val_nos_id as val_nos_id, + sum(current.val_balance - previous.val_balance + ifNull(withdrawals.withdrawn, 0)) AS real_balance_change + FROM ( + SELECT val_balance, val_id, val_nos_id + FROM validators_summary as curr + WHERE + val_nos_id IS NOT NULL AND + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS current + INNER JOIN ( + SELECT val_balance, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + epoch = (${epoch} - 1) + ${strIgnoreList} + LIMIT 1 BY val_id + ) AS previous ON previous.val_id = current.val_id + LEFT JOIN ( + SELECT + sum(val_balance_withdrawn) as withdrawn, val_id, val_nos_id + FROM ( + SELECT val_balance_withdrawn, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + val_balance_withdrawn > 0 AND + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) + GROUP BY val_id, val_nos_id + ) AS withdrawals + ON + withdrawals.val_id = current.val_id + GROUP BY current.val_nos_id + ) as bal ON att.val_nos_id = bal.val_nos_id + `; +}; export const avgChainRewardsAndPenaltiesStats = (epoch: Epoch): string => ` SELECT @@ -677,48 +824,55 @@ export const avgChainRewardsAndPenaltiesStats = (epoch: Epoch): string => ` ) as sync `; -export const userNodeOperatorsWithdrawalsStats = (epoch: Epoch): string => ` - SELECT - val_nos_id, - ifNull( - sumIf( - val_balance_withdrawn, - val_balance_withdrawn > 0 AND val_balance == 0 - ), - 0 - ) as full_withdrawn_sum, - ifNull( - sumIf( - val_balance_withdrawn, - val_balance_withdrawn > 0 AND val_balance != 0 - ), - 0 - ) as partial_withdrawn_sum, - ifNull( - countIf( - val_balance_withdrawn, - val_balance_withdrawn > 0 AND val_balance == 0 - ), - 0 - ) as full_withdrawn_count, - ifNull( - countIf( - val_balance_withdrawn, - val_balance_withdrawn > 0 AND val_balance != 0 - ), - 0 - ) as partial_withdrawn_count - FROM ( - SELECT val_balance_withdrawn, val_balance, val_id, val_nos_id - FROM validators_summary - WHERE - val_nos_id IS NOT NULL AND - val_balance_withdrawn > 0 AND - epoch = ${epoch} - LIMIT 1 BY val_id - ) - GROUP BY val_nos_id -`; +export const userNodeOperatorsWithdrawalsStats = (epoch: Epoch, ignoreList: string[]): string => { + let strIgnoreList = ''; + if (ignoreList.length > 0) { + strIgnoreList = `AND val_id NOT IN [${ignoreList.map((i) => `'${i}'`).join(',')}]`; + } + return ` + SELECT + val_nos_id, + ifNull( + sumIf( + val_balance_withdrawn, + val_balance_withdrawn > 0 AND val_balance == 0 + ), + 0 + ) as full_withdrawn_sum, + ifNull( + sumIf( + val_balance_withdrawn, + val_balance_withdrawn > 0 AND val_balance != 0 + ), + 0 + ) as partial_withdrawn_sum, + ifNull( + countIf( + val_balance_withdrawn, + val_balance_withdrawn > 0 AND val_balance == 0 + ), + 0 + ) as full_withdrawn_count, + ifNull( + countIf( + val_balance_withdrawn, + val_balance_withdrawn > 0 AND val_balance != 0 + ), + 0 + ) as partial_withdrawn_count + FROM ( + SELECT val_balance_withdrawn, val_balance, val_id, val_nos_id + FROM validators_summary + WHERE + val_nos_id IS NOT NULL AND + val_balance_withdrawn > 0 AND + epoch = ${epoch} + ${strIgnoreList} + LIMIT 1 BY val_id + ) + GROUP BY val_nos_id + `; +}; export const otherChainWithdrawalsStats = (epoch: Epoch): string => ` SELECT diff --git a/src/storage/clickhouse/clickhouse.service.ts b/src/storage/clickhouse/clickhouse.service.ts index 503b141a..c113746b 100644 --- a/src/storage/clickhouse/clickhouse.service.ts +++ b/src/storage/clickhouse/clickhouse.service.ts @@ -60,6 +60,7 @@ import migration_000002_rewards from './migrations/migration_000002_rewards'; import migration_000003_epoch_meta from './migrations/migration_000003_epoch_meta'; import migration_000004_epoch_processing from './migrations/migration_000004_epoch_processing'; import migration_000005_withdrawals from './migrations/migration_000005_withdrawals'; +import migration_000006_ignore_list from './migrations/migration_000006_ignore_list'; @Injectable() export class ClickhouseService implements OnModuleInit { @@ -68,7 +69,13 @@ export class ClickhouseService implements OnModuleInit { private readonly minBackoff: number; private readonly maxBackoff: number; private readonly chunkSize: number; + // List of validators indexes that should be ignored in the metrics private readonly retry: ReturnType; + public ignoreList: string[] = []; + + private async select(query: string): Promise { + return await (await this.retry(async () => await this.db.query({ query, format: 'JSONEachRow' }))).json(); + } public constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, @@ -97,14 +104,42 @@ export class ClickhouseService implements OnModuleInit { }); } - private async select(query: string): Promise { - return await (await this.retry(async () => await this.db.query({ query, format: 'JSONEachRow' }))).json(); - } - public async onModuleInit(): Promise { await this.retry(async () => await this.migrate()); } + public async setIndexesIgnoreListForMetrics(ignoreList: string[]): Promise { + if (JSON.stringify(this.ignoreList) == JSON.stringify(ignoreList)) { + return; + } + this.ignoreList = ignoreList; + const { amount } = (await this.select('SELECT count() as amount FROM validators_metrics_ignore_list'))[0]; + if (Number(amount) > 0) { + await this.retry( + async () => + await this.db.exec({ + query: `ALTER TABLE validators_metrics_ignore_list DELETE WHERE val_id`, + }), + ); + } + if (ignoreList.length > 0) { + await this.retry( + async () => + await this.db.insert({ + table: 'validators_metrics_ignore_list', + values: Stream.Readable.from( + ignoreList.map((i) => { + return { val_id: i }; + }), + { objectMode: true }, + ), + format: 'JSONEachRow', + }), + ); + } + this.logger.warn(`Successful setting of indexes ignore list for metrics. Indexes count [${ignoreList.length}]`); + } + public async getLastProcessedEpoch(): Promise { const data = ( await this.select( @@ -228,6 +263,7 @@ export class ClickhouseService implements OnModuleInit { migration_000003_epoch_meta, migration_000004_epoch_processing, migration_000005_withdrawals, + migration_000006_ignore_list, ]; for (const query of migrations) { await this.db.exec({ query }); @@ -235,21 +271,21 @@ export class ClickhouseService implements OnModuleInit { } public async getAvgValidatorBalanceDelta(epoch: Epoch): Promise { - return (await this.select(avgValidatorBalanceDelta(epoch))).map((v) => ({ + return (await this.select(avgValidatorBalanceDelta(epoch, this.ignoreList))).map((v) => ({ ...v, amount: Number(v.amount), })); } public async getValidatorQuantile0001BalanceDeltas(epoch: Epoch): Promise { - return (await this.select(validatorQuantile0001BalanceDeltasQuery(epoch))).map((v) => ({ + return (await this.select(validatorQuantile0001BalanceDeltasQuery(epoch, this.ignoreList))).map((v) => ({ ...v, amount: Number(v.amount), })); } public async getValidatorsCountWithNegativeDelta(epoch: Epoch): Promise { - return (await this.select(validatorsCountWithNegativeDeltaQuery(epoch))).map((v) => ({ + return (await this.select(validatorsCountWithNegativeDeltaQuery(epoch, this.ignoreList))).map((v) => ({ ...v, amount: Number(v.amount), })); @@ -259,7 +295,7 @@ export class ClickhouseService implements OnModuleInit { * Send query to Clickhouse and receives information about User Sync Committee participants */ public async getUserSyncParticipationAvgPercent(epoch: Epoch): Promise { - const ret = await this.select(userSyncParticipationAvgPercentQuery(epoch)); + const ret = await this.select(userSyncParticipationAvgPercentQuery(epoch, this.ignoreList)); return { amount: Number(ret[0].amount) }; } @@ -283,24 +319,28 @@ export class ClickhouseService implements OnModuleInit { * Send query to Clickhouse and receives information about Operator Sync Committee participants */ public async getOperatorSyncParticipationAvgPercents(epoch: Epoch): Promise { - return (await this.select(operatorsSyncParticipationAvgPercentsQuery(epoch))).map((v) => ({ - ...v, - amount: Number(v.amount), - })); + return (await this.select(operatorsSyncParticipationAvgPercentsQuery(epoch, this.ignoreList))).map( + (v) => ({ + ...v, + amount: Number(v.amount), + }), + ); } public async getValidatorsCountWithGoodSyncParticipationLastNEpoch( epoch: Epoch, epochInterval: number, chainAvg: number, - validatorIndexes: string[] = [], + ignoreList: string[], + neededIndexes: string[] = [], ): Promise { return ( await this.select( validatorsCountWithSyncParticipationByConditionLastNEpochQuery( epoch, epochInterval, - validatorIndexes, + ignoreList, + neededIndexes, `sync_percent >= (${chainAvg} - ${this.config.get('SYNC_PARTICIPATION_DISTANCE_DOWN_FROM_CHAIN_AVG')})`, ), ) @@ -318,14 +358,16 @@ export class ClickhouseService implements OnModuleInit { epoch: Epoch, epochInterval: number, chainAvg: number, - validatorIndexes: string[] = [], + ignoreList: string[], + neededIndexes: string[] = [], ): Promise { return ( await this.select( validatorsCountWithSyncParticipationByConditionLastNEpochQuery( epoch, epochInterval, - validatorIndexes, + ignoreList, + neededIndexes, `sync_percent < (${chainAvg} - ${this.config.get('SYNC_PARTICIPATION_DISTANCE_DOWN_FROM_CHAIN_AVG')})`, ), ) @@ -340,27 +382,48 @@ export class ClickhouseService implements OnModuleInit { epoch, 1, 'att_happened = 1 AND att_inc_delay = 1 AND att_valid_head = 1 AND att_valid_target = 1 AND att_valid_source = 1', + this.ignoreList, ); } public async getValidatorCountWithMissedAttestationsLastEpoch(epoch: Epoch) { - return await this.getValidatorCountByConditionAttestationsLastNEpoch(epoch, 1, 'att_happened = 0'); + return await this.getValidatorCountByConditionAttestationsLastNEpoch(epoch, 1, 'att_happened = 0', this.ignoreList); } public async getValidatorCountWithHighIncDelayAttestationsLastEpoch(epoch: Epoch) { - return await this.getValidatorCountByConditionAttestationsLastNEpoch(epoch, 1, 'att_happened = 1 AND att_inc_delay > 1'); + return await this.getValidatorCountByConditionAttestationsLastNEpoch( + epoch, + 1, + 'att_happened = 1 AND att_inc_delay > 1', + this.ignoreList, + ); } public async getValidatorCountWithInvalidHeadAttestationsLastEpoch(epoch: Epoch) { - return await this.getValidatorCountByConditionAttestationsLastNEpoch(epoch, 1, 'att_happened = 1 AND att_valid_head = 0'); + return await this.getValidatorCountByConditionAttestationsLastNEpoch( + epoch, + 1, + 'att_happened = 1 AND att_valid_head = 0', + this.ignoreList, + ); } public async getValidatorCountWithInvalidTargetAttestationsLastEpoch(epoch: Epoch) { - return await this.getValidatorCountByConditionAttestationsLastNEpoch(epoch, 1, 'att_happened = 1 AND att_valid_target = 0'); + return await this.getValidatorCountByConditionAttestationsLastNEpoch( + epoch, + 1, + 'att_happened = 1 AND att_valid_target = 0', + this.ignoreList, + ); } public async getValidatorCountWithInvalidSourceAttestationsLastEpoch(epoch: Epoch) { - return await this.getValidatorCountByConditionAttestationsLastNEpoch(epoch, 1, 'att_happened = 1 AND att_valid_source = 0'); + return await this.getValidatorCountByConditionAttestationsLastNEpoch( + epoch, + 1, + 'att_happened = 1 AND att_valid_source = 0', + this.ignoreList, + ); } public async getValidatorCountWithMissedAttestationsLastNEpoch(epoch: Epoch) { @@ -368,6 +431,7 @@ export class ClickhouseService implements OnModuleInit { epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), 'att_happened = 0', + this.ignoreList, ); } @@ -385,6 +449,7 @@ export class ClickhouseService implements OnModuleInit { epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), 'att_happened = 1 AND att_inc_delay > 1', + this.ignoreList, ); } @@ -393,6 +458,7 @@ export class ClickhouseService implements OnModuleInit { epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), 'att_happened = 1 AND att_inc_delay > 2', + this.ignoreList, ); } @@ -401,6 +467,7 @@ export class ClickhouseService implements OnModuleInit { epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), 'att_valid_head = 0', + this.ignoreList, ); } @@ -409,6 +476,7 @@ export class ClickhouseService implements OnModuleInit { epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), 'att_valid_target = 0', + this.ignoreList, ); } @@ -417,6 +485,7 @@ export class ClickhouseService implements OnModuleInit { epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), 'att_valid_source = 0', + this.ignoreList, ); } @@ -425,6 +494,7 @@ export class ClickhouseService implements OnModuleInit { epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), '(att_valid_head + att_valid_target + att_valid_source = 1)', + this.ignoreList, ); } @@ -436,11 +506,12 @@ export class ClickhouseService implements OnModuleInit { epoch: Epoch, epochInterval: number, condition: string, - validatorIndexes: string[] = [], + ignoreList: string[], + neededIndexes: string[] = [], ): Promise { return ( await this.select( - validatorCountByConditionAttestationLastNEpochQuery(epoch, epochInterval, validatorIndexes, condition), + validatorCountByConditionAttestationLastNEpochQuery(epoch, epochInterval, ignoreList, neededIndexes, condition), ) ).map((v) => ({ ...v, @@ -455,7 +526,7 @@ export class ClickhouseService implements OnModuleInit { public async getValidatorCountHighAvgIncDelayAttestationOfNEpochQuery(epoch: Epoch): Promise { return ( await this.select( - validatorCountHighAvgIncDelayAttestationOfNEpochQuery(epoch, this.config.get('BAD_ATTESTATION_EPOCHS')), + validatorCountHighAvgIncDelayAttestationOfNEpochQuery(epoch, this.config.get('BAD_ATTESTATION_EPOCHS'), this.ignoreList), ) ).map((v) => ({ ...v, @@ -465,11 +536,11 @@ export class ClickhouseService implements OnModuleInit { public async getValidatorsCountWithGoodProposes( epoch: Epoch, - validatorIndexes: string[] = [], + neededIndexes: string[] = [], ): Promise { return ( await this.select( - validatorsCountByConditionMissProposeQuery(epoch, validatorIndexes, 'block_proposed = 1'), + validatorsCountByConditionMissProposeQuery(epoch, this.ignoreList, neededIndexes, 'block_proposed = 1'), ) ).map((v) => ({ ...v, @@ -483,11 +554,11 @@ export class ClickhouseService implements OnModuleInit { */ public async getValidatorsCountWithMissedProposes( epoch: Epoch, - validatorIndexes: string[] = [], + neededIndexes: string[] = [], ): Promise { return ( await this.select( - validatorsCountByConditionMissProposeQuery(epoch, validatorIndexes, 'block_proposed = 0'), + validatorsCountByConditionMissProposeQuery(epoch, this.ignoreList, neededIndexes, 'block_proposed = 0'), ) ).map((v) => ({ ...v, @@ -496,12 +567,12 @@ export class ClickhouseService implements OnModuleInit { } public async getTotalBalance24hDifference(epoch: Epoch): Promise { - const ret = await this.select<{ amount }[]>(totalBalance24hDifferenceQuery(epoch)); + const ret = await this.select<{ amount }[]>(totalBalance24hDifferenceQuery(epoch, this.ignoreList)); if (ret[0]) return Number(ret[0].amount); } public async getOperatorBalance24hDifference(epoch: Epoch): Promise<{ val_nos_id; amount }[]> { - return (await this.select<{ val_nos_id; amount }[]>(operatorBalance24hDifferenceQuery(epoch))).map((v) => ({ + return (await this.select<{ val_nos_id; amount }[]>(operatorBalance24hDifferenceQuery(epoch, this.ignoreList))).map((v) => ({ ...v, amount: Number(v.amount), })); @@ -512,7 +583,7 @@ export class ClickhouseService implements OnModuleInit { * how many User Node Operator validators have active, slashed, pending status */ public async getUserNodeOperatorsStats(epoch: Epoch): Promise { - return (await this.select(userNodeOperatorsStatsQuery(epoch))).map((v) => ({ + return (await this.select(userNodeOperatorsStatsQuery(epoch, this.ignoreList))).map((v) => ({ ...v, active_ongoing: Number(v.active_ongoing), pending: Number(v.pending), @@ -527,7 +598,7 @@ export class ClickhouseService implements OnModuleInit { * how many User Node Operator validators have active, slashed, pending status */ public async getUserValidatorsSummaryStats(epoch: Epoch): Promise { - const ret = await this.select(userValidatorsSummaryStatsQuery(epoch)); + const ret = await this.select(userValidatorsSummaryStatsQuery(epoch, this.ignoreList)); return { active_ongoing: Number(ret[0].active_ongoing), pending: Number(ret[0].pending), @@ -557,7 +628,9 @@ export class ClickhouseService implements OnModuleInit { * User Node Operator proposes stats in the last N epochs */ public async getUserNodeOperatorsProposesStats(epoch: Epoch, epochInterval = 120): Promise { - return (await this.select(userNodeOperatorsProposesStatsLastNEpochQuery(epoch, epochInterval))).map((v) => ({ + return ( + await this.select(userNodeOperatorsProposesStatsLastNEpochQuery(epoch, epochInterval, this.ignoreList)) + ).map((v) => ({ ...v, all: Number(v.all), missed: Number(v.missed), @@ -613,7 +686,7 @@ export class ClickhouseService implements OnModuleInit { } public async getUserNodeOperatorsRewardsAndPenaltiesStats(epoch: Epoch): Promise { - return (await this.select(userNodeOperatorsRewardsAndPenaltiesStats(epoch))).map((v) => ({ + return (await this.select(userNodeOperatorsRewardsAndPenaltiesStats(epoch, this.ignoreList))).map((v) => ({ ...v, prop_reward: +v.prop_reward, prop_missed: +v.prop_missed, @@ -648,7 +721,7 @@ export class ClickhouseService implements OnModuleInit { } public async getUserNodeOperatorsWithdrawalsStats(epoch: Epoch): Promise { - return (await this.select(userNodeOperatorsWithdrawalsStats(epoch))).map((v) => ({ + return (await this.select(userNodeOperatorsWithdrawalsStats(epoch, this.ignoreList))).map((v) => ({ ...v, full_withdrawn_sum: +v.full_withdrawn_sum, full_withdrawn_count: +v.full_withdrawn_count, diff --git a/src/storage/clickhouse/migrations/migration_000006_ignore_list.ts b/src/storage/clickhouse/migrations/migration_000006_ignore_list.ts new file mode 100644 index 00000000..f0d5a1c0 --- /dev/null +++ b/src/storage/clickhouse/migrations/migration_000006_ignore_list.ts @@ -0,0 +1,8 @@ +const sql = ` +CREATE TABLE IF NOT EXISTS validators_metrics_ignore_list ( + "val_id" Int64 +) +ENGINE = ReplacingMergeTree() +ORDER BY val_id +`; +export default sql;