feat: withdrawals (#133)

* feat: withdrawals

* fix: block without withdrawals

* fix: metric

* fix: queries

* fix: import

src/common/eth-providers/consensus-provider/intefaces/response.interface.ts

@@ -55,10 +55,20 @@ export interface BlockInfoResponse {
       sync_aggregate: {
         sync_committee_bits: string;
       };
+      execution_payload: {
+        withdrawals: Withdrawal[];
+      };
     };
   };
 }
 
+export interface Withdrawal {
+  index: string;
+  validator_index: ValidatorIndex;
+  address: string;
+  amount: string;
+}
+
 export interface FinalityCheckpointsResponse {
   previous_justified: {
     epoch: string;

src/common/prometheus/prometheus.constants.ts

@@ -60,5 +60,10 @@ export const METRIC_AVG_CHAIN_MISSED_REWARD = `avg_chain_missed_reward`;
 export const METRIC_OPERATOR_MISSED_REWARD = `operator_missed_reward`;
 export const METRIC_AVG_CHAIN_PENALTY = `avg_chain_penalty`;
 export const METRIC_OPERATOR_PENALTY = `operator_penalty`;
+export const METRIC_OPERATOR_WITHDRAWALS_SUM = `operator_withdrawals_sum`;
+export const METRIC_OTHER_CHAIN_WITHDRAWALS_SUM = `other_chain_withdrawals_sum`;
+export const METRIC_OPERATOR_WITHDRAWALS_COUNT = `operator_withdrawals_count`;
+export const METRIC_OTHER_CHAIN_WITHDRAWALS_COUNT = `other_chain_withdrawals_count`;
+
 export const METRIC_CONTRACT_KEYS_TOTAL = `contract_keys_total`;
 export const METRIC_STETH_BUFFERED_ETHER_TOTAL = `steth_buffered_ether_total`;

src/common/prometheus/prometheus.service.ts

@@ -30,6 +30,10 @@ import {
   METRIC_OPERATOR_REAL_BALANCE_DELTA,
   METRIC_OPERATOR_REWARD,
   METRIC_OPERATOR_SYNC_PARTICIPATION_AVG_PERCENT,
+  METRIC_OPERATOR_WITHDRAWALS_COUNT,
+  METRIC_OPERATOR_WITHDRAWALS_SUM,
+  METRIC_OTHER_CHAIN_WITHDRAWALS_COUNT,
+  METRIC_OTHER_CHAIN_WITHDRAWALS_SUM,
   METRIC_OTHER_SYNC_PARTICIPATION_AVG_PERCENT,
   METRIC_OTHER_VALIDATOR_COUNT_GOOD_PROPOSE,
   METRIC_OTHER_VALIDATOR_COUNT_INVALID_ATTESTATION,
@@ -83,6 +87,8 @@ export enum PrometheusValStatus {
   Ongoing = 'ongoing',
   Pending = 'pending',
   Slashed = 'slashed',
+  WithdrawalPending = 'withdrawal_pending',
+  WithdrawalDone = 'withdrawal_done',
 }
 
 enum TaskStatus {
@@ -472,6 +478,30 @@ export class PrometheusService implements OnApplicationBootstrap {
     labelNames: ['nos_name', 'duty'],
   });
 
+  public operatorWithdrawalsSum = this.getOrCreateMetric('Gauge', {
+    name: METRIC_OPERATOR_WITHDRAWALS_SUM,
+    help: 'operator withdrawals sum',
+    labelNames: ['nos_name', 'type'],
+  });
+
+  public otherChainWithdrawalsSum = this.getOrCreateMetric('Gauge', {
+    name: METRIC_OTHER_CHAIN_WITHDRAWALS_SUM,
+    help: 'other chain withdrawals sum',
+    labelNames: ['type'],
+  });
+
+  public operatorWithdrawalsCount = this.getOrCreateMetric('Gauge', {
+    name: METRIC_OPERATOR_WITHDRAWALS_COUNT,
+    help: 'operator withdrawals count',
+    labelNames: ['nos_name', 'type'],
+  });
+
+  public otherChainWithdrawalsCount = this.getOrCreateMetric('Gauge', {
+    name: METRIC_OTHER_CHAIN_WITHDRAWALS_COUNT,
+    help: 'other chain withdrawals count',
+    labelNames: ['type'],
+  });
+
   public contractKeysTotal = this.getOrCreateMetric('Gauge', {
     name: METRIC_CONTRACT_KEYS_TOTAL,
     help: 'Contract keys',

src/duty/duty.metrics.ts

@@ -12,6 +12,7 @@ import { ProposeMetrics } from './propose';
 import { StateMetrics } from './state';
 import { SummaryMetrics } from './summary';
 import { SyncMetrics } from './sync';
+import { WithdrawalsMetrics } from './withdrawal';
 
 @Injectable()
 export class DutyMetrics {
@@ -25,14 +26,19 @@ export class DutyMetrics {
     protected readonly attestationMetrics: AttestationMetrics,
     protected readonly proposeMetrics: ProposeMetrics,
     protected readonly syncMetrics: SyncMetrics,
+    protected readonly withdrawalsMetrics: WithdrawalsMetrics,
     protected readonly summaryMetrics: SummaryMetrics,
     protected readonly storage: ClickhouseService,
   ) {}
 
   @TrackTask('calc-all-duties-metrics')
   public async calculate(epoch: Epoch, possibleHighRewardValidators: string[]): Promise<any> {
     this.logger.log('Calculating duties metrics of user validators');
-    await Promise.all([this.withPossibleHighReward(epoch, possibleHighRewardValidators), this.stateMetrics.calculate(epoch)]);
+    await Promise.all([
+      this.withPossibleHighReward(epoch, possibleHighRewardValidators),
+      this.stateMetrics.calculate(epoch),
+      this.withdrawalsMetrics.calculate(epoch),
+    ]);
     // we must calculate summary metrics after all duties to avoid errors in processing
     await this.summaryMetrics.calculate(epoch);
     await this.storage.updateEpochProcessing({ epoch, is_calculated: true });

src/duty/duty.module.ts

@@ -13,13 +13,15 @@ import { ProposeModule } from './propose';
 import { StateModule } from './state';
 import { SummaryModule } from './summary';
 import { SyncModule } from './sync';
+import { WithdrawalsModule } from './withdrawal';
 
 @Module({
   imports: [
     AttestationModule,
     ProposeModule,
     StateModule,
     SyncModule,
+    WithdrawalsModule,
     SummaryModule,
     ConsensusProviderModule,
     BlockCacheModule,

src/duty/duty.service.ts

@@ -19,6 +19,7 @@ import { StateService } from './state';
 import { SummaryService } from './summary';
 import { SyncService } from './sync';
 import { syncReward } from './sync/sync.constants';
+import { WithdrawalsService } from './withdrawal';
 
 @Injectable()
 export class DutyService {
@@ -36,6 +37,7 @@ export class DutyService {
     protected readonly summary: SummaryService,
     protected readonly storage: ClickhouseService,
     protected readonly rewards: DutyRewards,
+    protected readonly withdrawals: WithdrawalsService,
   ) {}
 
   public async checkAndWrite({ epoch, stateSlot }: { epoch: Epoch; stateSlot: Slot }): Promise<string[]> {
@@ -65,6 +67,7 @@ export class DutyService {
       this.attestation.check(epoch, stateSlot),
       this.sync.check(epoch, stateSlot),
       this.propose.check(epoch),
+      this.withdrawals.check(epoch),
     ]);
     // must be done after all duties check
     await this.fillCurrentEpochMetadata(epoch);

src/duty/state/state.metrics.ts

@@ -81,6 +81,24 @@ export class StateMetrics {
       },
       (item) => item.pending,
     );
+    setUserOperatorsMetric(
+      this.prometheus.userValidators,
+      data,
+      this.operators,
+      {
+        status: PrometheusValStatus.WithdrawalPending,
+      },
+      (item) => item.withdraw_pending,
+    );
+    setUserOperatorsMetric(
+      this.prometheus.userValidators,
+      data,
+      this.operators,
+      {
+        status: PrometheusValStatus.WithdrawalDone,
+      },
+      (item) => item.withdrawn,
+    );
   }
 
   private async userValidatorsStats() {
@@ -107,6 +125,20 @@ export class StateMetrics {
       },
       result.pending,
     );
+    this.prometheus.validators.set(
+      {
+        owner: Owner.USER,
+        status: PrometheusValStatus.WithdrawalPending,
+      },
+      result.withdraw_pending,
+    );
+    this.prometheus.validators.set(
+      {
+        owner: Owner.USER,
+        status: PrometheusValStatus.WithdrawalDone,
+      },
+      result.withdrawn,
+    );
   }
 
   private async otherValidatorsStats() {
@@ -133,6 +165,20 @@ export class StateMetrics {
       },
       result.slashed,
     );
+    this.prometheus.validators.set(
+      {
+        owner: Owner.OTHER,
+        status: PrometheusValStatus.WithdrawalPending,
+      },
+      result.withdraw_pending,
+    );
+    this.prometheus.validators.set(
+      {
+        owner: Owner.OTHER,
+        status: PrometheusValStatus.WithdrawalDone,
+      },
+      result.withdrawn,
+    );
   }
 
   private async avgDeltas() {

src/duty/summary/summary.service.ts

@@ -18,6 +18,7 @@ export interface ValidatorDutySummary {
   val_slashed?: boolean;
   val_status?: ValStatus;
   val_balance?: bigint;
+  val_balance_withdrawn?: bigint;
   val_effective_balance?: bigint;
   ///
   is_proposer?: boolean;

src/duty/withdrawal/index.ts

@@ -0,0 +1,3 @@
+export * from './withdrawals.module';
+export * from './withdrawals.service';
+export * from './withdrawals.metrics';

src/duty/withdrawal/withdrawals.metrics.ts

@@ -0,0 +1,74 @@
+import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
+import { Inject, Injectable, LoggerService } from '@nestjs/common';
+
+import { ConfigService } from 'common/config';
+import { Epoch } from 'common/eth-providers/consensus-provider/types';
+import { PrometheusService, TrackTask, setUserOperatorsMetric } from 'common/prometheus';
+import { RegistryService, RegistrySourceOperator } from 'common/validators-registry';
+import { ClickhouseService } from 'storage/clickhouse';
+
+enum WithdrawalType {
+  Partial = 'partial',
+  Full = 'full',
+}
+
+@Injectable()
+export class WithdrawalsMetrics {
+  protected processedEpoch: number;
+  protected operators: RegistrySourceOperator[];
+  public constructor(
+    @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
+    protected readonly config: ConfigService,
+    protected readonly prometheus: PrometheusService,
+    protected readonly registryService: RegistryService,
+    protected readonly storage: ClickhouseService,
+  ) {}
+
+  @TrackTask('calc-withdrawals-metrics')
+  public async calculate(epoch: Epoch) {
+    this.logger.log('Calculating withdrawals metrics');
+    this.processedEpoch = epoch;
+    this.operators = this.registryService.getOperators();
+    await Promise.all([this.userNodeOperatorsWithdrawalsStats(), this.otherChainWithdrawalsStats()]);
+  }
+
+  private async userNodeOperatorsWithdrawalsStats() {
+    const data = await this.storage.getUserNodeOperatorsWithdrawalsStats(this.processedEpoch);
+    setUserOperatorsMetric(
+      this.prometheus.operatorWithdrawalsSum,
+      data,
+      this.operators,
+      { type: WithdrawalType.Partial },
+      (item) => item.partial_withdrawn_sum,
+    );
+    setUserOperatorsMetric(
+      this.prometheus.operatorWithdrawalsSum,
+      data,
+      this.operators,
+      { type: WithdrawalType.Full },
+      (item) => item.full_withdrawn_sum,
+    );
+    setUserOperatorsMetric(
+      this.prometheus.operatorWithdrawalsCount,
+      data,
+      this.operators,
+      { type: WithdrawalType.Partial },
+      (item) => item.partial_withdrawn_count,
+    );
+    setUserOperatorsMetric(
+      this.prometheus.operatorWithdrawalsCount,
+      data,
+      this.operators,
+      { type: WithdrawalType.Full },
+      (item) => item.full_withdrawn_count,
+    );
+  }
+
+  private async otherChainWithdrawalsStats() {
+    const result = await this.storage.getOtherChainWithdrawalsStats(this.processedEpoch);
+    this.prometheus.otherChainWithdrawalsSum.set({ type: WithdrawalType.Partial }, result.partial_withdrawn_sum);
+    this.prometheus.otherChainWithdrawalsSum.set({ type: WithdrawalType.Full }, result.full_withdrawn_sum);
+    this.prometheus.otherChainWithdrawalsCount.set({ type: WithdrawalType.Partial }, result.partial_withdrawn_count);
+    this.prometheus.otherChainWithdrawalsCount.set({ type: WithdrawalType.Full }, result.full_withdrawn_count);
+  }
+}

src/duty/withdrawal/withdrawals.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+
+import { ConsensusProviderModule } from 'common/eth-providers';
+import { RegistryModule } from 'common/validators-registry';
+import { ClickhouseModule } from 'storage/clickhouse';
+
+import { SummaryModule } from '../summary';
+import { WithdrawalsMetrics } from './withdrawals.metrics';
+import { WithdrawalsService } from './withdrawals.service';
+
+@Module({
+  imports: [RegistryModule, ConsensusProviderModule, ClickhouseModule, SummaryModule],
+  providers: [WithdrawalsService, WithdrawalsMetrics],
+  exports: [WithdrawalsService, WithdrawalsMetrics],
+})
+export class WithdrawalsModule {}

src/duty/withdrawal/withdrawals.service.ts

@@ -0,0 +1,45 @@
+import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
+import { Inject, Injectable, LoggerService } from '@nestjs/common';
+
+import { ConfigService } from 'common/config';
+import { BlockInfoResponse, ConsensusProviderService } from 'common/eth-providers';
+import { Epoch } from 'common/eth-providers/consensus-provider/types';
+import { PrometheusService, TrackTask } from 'common/prometheus';
+import { RegistryService } from 'common/validators-registry';
+import { ClickhouseService } from 'storage/clickhouse';
+
+import { range } from '../../common/functions/range';
+import { SummaryService } from '../summary';
+
+@Injectable()
+export class WithdrawalsService {
+  public constructor(
+    @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
+    protected readonly config: ConfigService,
+    protected readonly prometheus: PrometheusService,
+    protected readonly clClient: ConsensusProviderService,
+    protected readonly summary: SummaryService,
+    protected readonly storage: ClickhouseService,
+    protected readonly registry: RegistryService,
+  ) {}
+
+  @TrackTask('check-withdrawals')
+  public async check(epoch: Epoch): Promise<void> {
+    this.logger.log('Getting withdrawals for epoch');
+    const slotsInEpoch = this.config.get('FETCH_INTERVAL_SLOTS');
+    const firstSlotInEpoch = epoch * slotsInEpoch;
+    const slots: number[] = range(firstSlotInEpoch, firstSlotInEpoch + slotsInEpoch);
+    const toFetch = slots.map((s) => this.clClient.getBlockInfo(s));
+    const blocks = (await Promise.all(toFetch)).filter((b) => b != undefined) as BlockInfoResponse[];
+    for (const block of blocks) {
+      const withdrawals = block.message.body.execution_payload.withdrawals ?? [];
+      for (const withdrawal of withdrawals) {
+        this.summary.epoch(epoch).set({
+          epoch,
+          val_id: Number(withdrawal.validator_index),
+          val_balance_withdrawn: BigInt(withdrawal.amount),
+        });
+      }
+    }
+  }
+}