New test: call and check responses of all calls made in Performance Cron

docker-compose-dev.yml

@@ -4,7 +4,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.brain.dev
-    container_name: brain
+    container_name: DAppNodePackage-brain.web3signer-holesky.dnp.dappnode.eth
     volumes:
       - .:/app
     environment:

packages/brain/src/modules/apiClients/beaconchain/index.ts

@@ -144,7 +144,7 @@ export class BeaconchainApi extends StandardApi {
   }): Promise<BeaconchainValidatorFromStateGetResponse> {
     try {
       return await this.request({
-        method: "POST",
+        method: "GET",
         endpoint: path.join(this.beaconchainEndpoint, "states", state, "validators", pubkey)
       });
     } catch (e) {

packages/brain/src/modules/apiClients/beaconchain/types.ts

@@ -48,18 +48,20 @@ export interface TotalRewards {
   inactivity: string;
 }
 
+export interface IdealRewards {
+  effective_balance: string;
+  head: string;
+  target: string;
+  source: string;
+  inclusion_delay: string;
+  inactivity: string;
+}
+
 export interface BeaconchainAttestationRewardsPostResponse {
   execution_optimistic: boolean;
   finalized: boolean;
   data: {
-    ideal_rewards: {
-      effective_balance: string;
-      head: string;
-      target: string;
-      source: string;
-      inclusion_delay: string;
-      inactivity: string;
-    }[];
+    ideal_rewards: IdealRewards[];
     total_rewards: TotalRewards[];
   };
 }

packages/brain/src/modules/apiClients/postgres/index.ts

@@ -1,54 +1,8 @@
 import postgres from "postgres";
 import logger from "../../logger/index.js";
-import { BlockProposalStatus, ValidatorPerformance } from "./types.js";
+import { BlockProposalStatus, Columns, ValidatorPerformance, ValidatorPerformancePostgres } from "./types.js";
 import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
 
-// Postgres has a built in class for errors PostgresError. i.e:
-// PostgresError: type "block_proposal_status" already exists
-//     at ErrorResponse (file:///app/node_modules/postgres/src/connection.js:788:26)
-//     at handle (file:///app/node_modules/postgres/src/connection.js:474:6)
-//     at Socket.data (file:///app/node_modules/postgres/src/connection.js:315:9)
-//     at Socket.emit (node:events:519:28)
-//     at addChunk (node:internal/streams/readable:559:12)
-//     at readableAddChunkPushByteMode (node:internal/streams/readable:510:3)
-//     at Readable.push (node:internal/streams/readable:390:5)
-//     at TCP.onStreamRead (node:internal/stream_base_commons:191:23) {
-//   severity_local: 'ERROR',
-//   severity: 'ERROR',
-//   code: '42710',
-//   where: `SQL statement "CREATE TYPE BLOCK_PROPOSAL_STATUS AS ENUM('Missed', 'Proposed', 'Unchosen')"\n` +
-//     'PL/pgSQL function inline_code_block line 5 at SQL statement',
-//   file: 'typecmds.c',
-//   line: '1170',
-//   routine: 'DefineEnum'
-// }
-
-enum Columns {
-  validatorIndex = "validator_index",
-  epoch = "epoch",
-  executionClient = "execution_client",
-  consensusClient = "consensus_client",
-  slot = "slot",
-  liveness = "liveness",
-  blockProposalStatus = "block_proposal_status",
-  syncCommitteeRewards = "sync_comittee_rewards",
-  attestationsTotalRewards = "attestations_total_rewards",
-  error = "error"
-}
-
-interface ValidatorPerformancePostgres {
-  [Columns.validatorIndex]: number;
-  [Columns.epoch]: number;
-  [Columns.executionClient]: ExecutionClient;
-  [Columns.consensusClient]: ConsensusClient;
-  [Columns.slot]: number;
-  [Columns.liveness]: boolean;
-  [Columns.blockProposalStatus]: BlockProposalStatus;
-  [Columns.syncCommitteeRewards]: number;
-  [Columns.attestationsTotalRewards]: string;
-  [Columns.error]: string;
-}
-
 export class PostgresClient {
   private readonly tableName = "validators_performance";
   private readonly BLOCK_PROPOSAL_STATUS = "BLOCK_PROPOSAL_STATUS";
@@ -83,17 +37,7 @@ SELECT pg_total_relation_size('${this.tableName}');
   }
 
   /**
-   * Initializes the database by creating the required table if it does not exist.
-   * The table will have the following columns:
-   * - validator_index: The index of the validator.
-   * - epoch: The epoch number.
-   * - slot: The slot number.
-   * - liveness: The liveness status of the validator.
-   * - block_proposal_status: The status of the block proposal (missed, proposed, unchosen).
-   * - sync_comittee_rewards: The rewards received by the validator for participating in the sync committee.
-   * - attestations_rewards: The rewards received by the validator for participating in the attestations.
-   * - error: Any error message related to the validator's performance fetch.
-   * The primary key will be a combination of validator_index and epoch.
+   * Initializes the database by creating the required table if it does not exist with the required columns.
    */
   public async initialize() {
     // important: enum create types must be broken into separate conditional checks for each ENUM type before trying to create it.
@@ -106,7 +50,6 @@ SELECT pg_total_relation_size('${this.tableName}');
         WHEN duplicate_object THEN NULL;
     END $$;
   `);
-
     // Check and create EXECUTION_CLIENT ENUM type if not exists
     await this.sql.unsafe(`
     DO $$
@@ -139,6 +82,7 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} (
   ${Columns.blockProposalStatus} ${this.BLOCK_PROPOSAL_STATUS},
   ${Columns.syncCommitteeRewards} BIGINT,
   ${Columns.attestationsTotalRewards} JSONB,
+  ${Columns.attestationsIdealRewards} JSONB,
   ${Columns.error} JSONB NULL,
   PRIMARY KEY (${Columns.validatorIndex}, ${Columns.epoch})
 );
@@ -180,6 +124,7 @@ DO UPDATE SET
   ${Columns.blockProposalStatus} = EXCLUDED.${Columns.blockProposalStatus},
   ${Columns.syncCommitteeRewards} = EXCLUDED.${Columns.syncCommitteeRewards},
   ${Columns.attestationsTotalRewards} = EXCLUDED.${Columns.attestationsTotalRewards},
+  ${Columns.attestationsIdealRewards} = EXCLUDED.${Columns.attestationsIdealRewards},
   ${Columns.error} = EXCLUDED.${Columns.error};
     `;
 
@@ -194,6 +139,7 @@ DO UPDATE SET
       data.blockProposalStatus ?? null,
       data.syncCommitteeRewards ?? null,
       data.attestationsTotalRewards ? JSON.stringify(data.attestationsTotalRewards) : null, // JSONB expects a string or null
+      data.attestationsIdealRewards ? JSON.stringify(data.attestationsIdealRewards) : null, // JSONB expects a string or null
       data.error ? JSON.stringify(data.error) : null // JSONB expects a string or null
     ]);
   }
@@ -222,6 +168,7 @@ WHERE ${Columns.validatorIndex} = ANY($1)
       blockProposalStatus: row.block_proposal_status,
       syncCommitteeRewards: row.sync_comittee_rewards,
       attestationsTotalRewards: JSON.parse(row.attestations_total_rewards),
+      attestationsIdealRewards: JSON.parse(row.attestations_ideal_rewards),
       error: JSON.parse(row.error)
     }));
   }
@@ -270,6 +217,7 @@ AND ${Columns.epoch} <= $3
         blockProposalStatus: row.block_proposal_status,
         syncCommitteeRewards: row.sync_comittee_rewards,
         attestationsTotalRewards: JSON.parse(row.attestations_total_rewards),
+        attestationsIdealRewards: JSON.parse(row.attestations_ideal_rewards),
         error: JSON.parse(row.error)
       };
 

packages/brain/src/modules/apiClients/postgres/types.ts

@@ -1,4 +1,34 @@
 import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
+import { IdealRewards, TotalRewards } from "../beaconchain/types.js";
+
+export enum Columns {
+  validatorIndex = "validator_index",
+  epoch = "epoch",
+  executionClient = "execution_client",
+  consensusClient = "consensus_client",
+  slot = "slot",
+  liveness = "liveness",
+  blockProposalStatus = "block_proposal_status",
+  syncCommitteeRewards = "sync_comittee_rewards",
+  attestationsTotalRewards = "attestations_total_rewards",
+  attestationsIdealRewards = "attestations_ideal_rewards",
+  error = "error"
+}
+
+// Interface data write with Postgres client
+export interface ValidatorPerformancePostgres {
+  [Columns.validatorIndex]: number;
+  [Columns.epoch]: number;
+  [Columns.executionClient]: ExecutionClient;
+  [Columns.consensusClient]: ConsensusClient;
+  [Columns.slot]: number;
+  [Columns.liveness]: boolean;
+  [Columns.blockProposalStatus]: BlockProposalStatus;
+  [Columns.syncCommitteeRewards]: number;
+  [Columns.attestationsTotalRewards]: string;
+  [Columns.attestationsIdealRewards]: string;
+  [Columns.error]: string;
+}
 
 export enum BlockProposalStatus {
   Missed = "Missed",
@@ -7,22 +37,15 @@ export enum BlockProposalStatus {
   Error = "Error"
 }
 
-export interface AttestationsTotalRewards {
-  validator_index: string;
-  head: string;
-  target: string;
-  source: string;
-  inclusion_delay: string;
-  inactivity: string;
-}
-
+// Interface data return from Postgres client
 export interface ValidatorPerformance {
   validatorIndex: number;
   epoch: number;
   executionClient: ExecutionClient;
   consensusClient: ConsensusClient;
   blockProposalStatus?: BlockProposalStatus;
-  attestationsTotalRewards?: AttestationsTotalRewards;
+  attestationsTotalRewards?: TotalRewards;
+  attestationsIdealRewards?: IdealRewards;
   slot?: number;
   liveness?: boolean;
   syncCommitteeRewards?: number;

packages/brain/src/modules/cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.ts

@@ -6,9 +6,8 @@ import logger from "../../logger/index.js";
 import { logPrefix } from "./logPrefix.js";
 
 /**
- * Get the active validators from the beaconchain API. To do so it will get the validator indexes from the brain db,
- * if there are no indexes, it will get them from the beaconchain API and update the brain db with the indexes for further
- * use.
+ * Get the active validators from the beaconchain API. To do so, it will get the validator indexes from the brain db.
+ * If there are no indexes, it will get them from the beaconchain API and update the brain db with the indexes for further use.
  *
  * @param {BeaconchainApi} beaconchainApi - Beaconchain API client.
  * @param {BrainDataBase} brainDb - Brain DB client.
@@ -23,15 +22,15 @@ export async function getActiveValidatorsLoadedInBrain({
 }): Promise<string[]> {
   const validatorIndexes = await getValidatorIndexesAndSaveInDb({ beaconchainApi, brainDb });
   if (validatorIndexes.length === 0) return [];
-  return (
-    await beaconchainApi.postStateValidators({
-      body: {
-        ids: validatorIndexes,
-        statuses: [ValidatorStatus.ACTIVE_ONGOING]
-      },
-      stateId: "finalized"
-    })
-  ).data.map((validator) => validator.index.toString());
+  const response = await beaconchainApi.postStateValidators({
+    body: {
+      ids: validatorIndexes,
+      statuses: [ValidatorStatus.ACTIVE_ONGOING]
+    },
+    stateId: "finalized"
+  });
+
+  return response.data.map((validator) => validator.index.toString());
 }
 
 /**
@@ -52,24 +51,34 @@ async function getValidatorIndexesAndSaveInDb({
   const brainDbData = brainDb.getData();
   if (isEmpty(brainDbData)) return [];
 
-  // get validator indexes from brain db
+  // Get validator indexes from brain db
   const validatorIndexes: string[] = [];
   const validatorPubkeysWithNoIndex: string[] = [];
-  // iterate over brain db data and push the indexes to the array
+  // Iterate over brain db data and push the indexes to the array
   for (const [pubkey, details] of Object.entries(brainDbData)) {
     if (details.index) validatorIndexes.push(details.index.toString());
     else validatorPubkeysWithNoIndex.push(pubkey);
   }
 
+  // If there are validators with no index, fetch them in batch from the beaconchain API
   if (validatorPubkeysWithNoIndex.length > 0) {
     logger.debug(`${logPrefix}Getting validator indexes from pubkeys`);
-    await Promise.all(
-      validatorPubkeysWithNoIndex.map(async (pubkey) => {
-        const index = (await beaconchainApi.getStateValidator({ state: "finalized", pubkey })).data.index;
-        validatorIndexes.push(index);
-        brainDb.updateValidators({ validators: { [pubkey]: { ...brainDbData[pubkey], index: parseInt(index) } } });
-      })
-    );
+
+    const response = await beaconchainApi.postStateValidators({
+      stateId: "finalized",
+      body: {
+        ids: validatorPubkeysWithNoIndex,
+        statuses: []
+      }
+    });
+
+    // Update the brain DB with fetched indexes and add them to the array
+    for (const validatorData of response.data) {
+      const { pubkey } = validatorData.validator;
+      const index = validatorData.index;
+      validatorIndexes.push(index);
+      brainDb.updateValidators({ validators: { [pubkey]: { ...brainDbData[pubkey], index: parseInt(index) } } });
+    }
   }
 
   return validatorIndexes;

packages/brain/src/modules/cron/trackValidatorsPerformance/getValidatorAttestationsRewards.ts

@@ -0,0 +1,29 @@
+import { BeaconchainApi } from "../../apiClients/index.js";
+import { IdealRewards, TotalRewards } from "../../apiClients/types.js";
+
+/**
+ * Get attestations rewards for the validators:
+ * - Total rewards
+ * - Ideal reward last item from array of IdealRewards -> is the ideal reward maximum that could be achieved by the validator
+ */
+export async function getValidatorAttestationsRewards({
+  beaconchainApi,
+  epoch,
+  activeValidatorsIndexes
+}: {
+  beaconchainApi: BeaconchainApi;
+  epoch: string;
+  activeValidatorsIndexes: string[];
+}): Promise<{ totalRewards: TotalRewards[]; idealRewards: IdealRewards }> {
+  const attestationsRewards = (
+    await beaconchainApi.getAttestationsRewards({
+      epoch,
+      pubkeysOrIndexes: activeValidatorsIndexes
+    })
+  ).data;
+
+  return {
+    totalRewards: attestationsRewards.total_rewards,
+    idealRewards: attestationsRewards.ideal_rewards[attestationsRewards.ideal_rewards.length - 1]
+  };
+}