Implement indexer api

packages/brain/src/index.ts

@@ -51,7 +51,7 @@ reloadValidatorsCronTask.start();
 trackValidatorsPerformanceCronTask.start();
 
 // Start server APIs
-const { uiServer, launchpadServer, brainApiServer } = getServers({
+const { uiServer, launchpadServer, brainApiServer, indexerApi } = getServers({
   brainConfig: config,
   uiBuildPath: path.resolve(__dirname, params.uiBuildDirName),
   signerApi,
@@ -73,6 +73,7 @@ function handle(signal: string): void {
   uiServer.close();
   launchpadServer.close();
   brainApiServer.close();
+  indexerApi.close();
   logger.debug(`Stopped all cron jobs and closed all connections.`);
   process.exit(0);
 }

packages/brain/src/modules/apiClients/postgres/index.ts

@@ -1,6 +1,6 @@
 import postgres from "postgres";
 import logger from "../../logger/index.js";
-import { EpochsValidatorsMap, DataPerEpoch, Columns, ValidatorsDataPerEpochMap, PostgresDataRow } from "./types.js";
+import { DataPerEpoch, Columns, ValidatorsDataPerEpochMap, PostgresDataRow, EpochsValidatorsData } from "./types.js";
 
 export class PostgresClient {
   private readonly tableName = "epochs_data";
@@ -99,7 +99,7 @@ ${Columns.error} = EXCLUDED.${Columns.error}
     validatorIndexes: string[];
     startEpoch: number;
     endEpoch: number;
-  }): Promise<EpochsValidatorsMap> {
+  }): Promise<EpochsValidatorsData> {
     const query = `
 SELECT * FROM ${this.tableName}
 WHERE ${Columns.validatorindex} = ANY($1)
@@ -109,7 +109,7 @@ AND ${Columns.epoch} <= $3
 
     const result: PostgresDataRow[] = await this.sql.unsafe(query, [validatorIndexes, startEpoch, endEpoch]);
 
-    const epochsValidatorsMap: EpochsValidatorsMap = new Map();
+    const epochsValidatorData: EpochsValidatorsData = Object.create(null);
 
     for (const row of result) {
       const { validatorindex, epoch, clients, attestation, block, synccommittee, slot, error } = row;
@@ -124,12 +124,12 @@ AND ${Columns.epoch} <= $3
         [Columns.error]: error
       };
 
-      if (!epochsValidatorsMap.has(epochNumber)) epochsValidatorsMap.set(epochNumber, new Map());
+      if (!epochsValidatorData[epochNumber]) epochsValidatorData[epochNumber] = Object.create(null);
 
-      epochsValidatorsMap.get(epochNumber)!.set(validatorIndexNumber, data);
+      epochsValidatorData[epochNumber][validatorIndexNumber] = data;
     }
 
-    return epochsValidatorsMap;
+    return epochsValidatorData;
   }
 
   /**

packages/brain/src/modules/apiClients/postgres/types.ts

@@ -13,8 +13,11 @@ export enum Columns {
   error = "error"
 }
 
-// Indexed by epoch number
-export type EpochsValidatorsMap = Map<number, ValidatorsDataPerEpochMap>;
+export interface EpochsValidatorsData {
+  [epoch: number]: {
+    [validatorIndex: number]: DataPerEpoch;
+  };
+}
 
 // Indexed by validator index
 export type ValidatorsDataPerEpochMap = Map<number, DataPerEpoch>;

packages/brain/src/modules/apiServers/index.ts

@@ -10,6 +10,7 @@ import { BeaconchainApi } from "../apiClients/beaconchain/index.js";
 import { CronJob } from "../cron/cron.js";
 import { startLaunchpadApi } from "./launchpad/index.js";
 import http from "http";
+import { startIndexerApi } from "./indexer/index.js";
 
 export const getServers = ({
   brainConfig,
@@ -35,6 +36,7 @@ export const getServers = ({
   uiServer: http.Server;
   launchpadServer: http.Server;
   brainApiServer: http.Server;
+  indexerApi: http.Server;
 } => {
   return {
     uiServer: startUiServer({
@@ -59,6 +61,10 @@ export const getServers = ({
     }),
     brainApiServer: startBrainApi({
       brainDb
+    }),
+    indexerApi: startIndexerApi({
+      brainDb,
+      postgresClient
     })
   };
 };

packages/brain/src/modules/apiServers/indexer/config.ts

@@ -0,0 +1,3 @@
+export const corsOptions = {
+  origin: ["http://csm-report-indexer.dappnode", "http://csm-report-indexer.testnet.dappnode"] // TODO: update with DAppNodePackage-lido-csm.dnp.dappnode.eth domains
+};

packages/brain/src/modules/apiServers/indexer/index.ts

@@ -0,0 +1 @@
+export { startIndexerApi } from "./startIndexerApi.js";

packages/brain/src/modules/apiServers/indexer/routes/epochs/index.ts

@@ -0,0 +1 @@
+export { createIndexerEpochsRouter } from "./route.js";

packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts

@@ -0,0 +1,126 @@
+import express from "express";
+import logger from "../../../../logger/index.js";
+import { validateQueryParams } from "./validation.js";
+import { RequestParsed } from "./types.js";
+import { BrainDataBase } from "../../../../db/index.js";
+import { PostgresClient } from "../../../../apiClients/index.js";
+import { Tag, tags } from "@stakingbrain/common";
+import { DataPerEpoch } from "../../../../apiClients/types.js";
+import { StakingBrainDb } from "../../../../db/types.js";
+
+interface EpochsTagsValidatorsData {
+  [epoch: number]: {
+    [tag: string]: {
+      [validatorIndex: number]: DataPerEpoch;
+    };
+  };
+}
+
+export const createIndexerEpochsRouter = ({
+  postgresClient,
+  brainDb
+}: {
+  postgresClient: PostgresClient;
+  brainDb: BrainDataBase;
+}) => {
+  const epochsRouter = express.Router();
+  const epochsEndpoint = "/api/v0/indexer/epochs";
+
+  epochsRouter.get(epochsEndpoint, validateQueryParams, async (req: RequestParsed, res) => {
+    const { start, end, tag } = req.query;
+
+    try {
+      const brainDbData = brainDb.getData();
+      const validatorsTagIndexesMap = getValidatorsTagsIndexesMap({ brainDbData, tag });
+
+      // If validatorIndexes is empty, return empty object
+      if (validatorsTagIndexesMap.size === 0) {
+        res.send({});
+        return;
+      }
+
+      const epochsTagsValidatorsData = await getEpochsTagsValidatorsData({
+        postgresClient,
+        validatorsTagIndexesMap,
+        start,
+        end
+      });
+
+      res.send(epochsTagsValidatorsData);
+    } catch (e) {
+      logger.error(e);
+      res.status(500).send({ message: "Internal server error" });
+    }
+  });
+
+  return epochsRouter;
+};
+
+function getValidatorsTagsIndexesMap({
+  brainDbData,
+  tag
+}: {
+  brainDbData: StakingBrainDb;
+  tag?: Tag[];
+}): Map<Tag, number[]> {
+  const validatorsTagIndexesMap = new Map<Tag, number[]>([["solo", [1234]]]);
+
+  // Filter the tags to consider, based on whether the `tag` parameter is provided
+  const tagsToConsider = tag ? tag : tags;
+
+  // Initialize an empty array for each tag in the map
+  for (const t of tagsToConsider) validatorsTagIndexesMap.set(t, []);
+
+  // Iterate over brainDbData to populate the map with the indexes
+  for (const [_, details] of Object.entries(brainDbData)) {
+    const validatorTag = details.tag;
+    const validatorIndex = details.index;
+
+    // Check if the validator tag is in the tags to consider and if an index exists
+    if (tagsToConsider.includes(validatorTag) && validatorIndex !== undefined) {
+      // Initialize the array if it doesn't exist in the map
+      if (!validatorsTagIndexesMap.has(validatorTag)) {
+        validatorsTagIndexesMap.set(validatorTag, []);
+      }
+
+      // Push the validator's index into the corresponding array
+      validatorsTagIndexesMap.get(validatorTag)!.push(validatorIndex);
+    }
+  }
+
+  return validatorsTagIndexesMap;
+}
+
+async function getEpochsTagsValidatorsData({
+  postgresClient,
+  validatorsTagIndexesMap,
+  start,
+  end
+}: {
+  postgresClient: PostgresClient;
+  validatorsTagIndexesMap: Map<Tag, number[]>;
+  start: number;
+  end: number;
+}): Promise<EpochsTagsValidatorsData> {
+  const epochsTagsValidatorsData: EpochsTagsValidatorsData = Object.create(null);
+
+  // Get epochs data for each tag
+  for (const [tag, indexes] of validatorsTagIndexesMap) {
+    const epochsValidatorsData = await postgresClient.getEpochsDataMapForEpochRange({
+      validatorIndexes: indexes.map((index) => index.toString()),
+      startEpoch: start,
+      endEpoch: end
+    });
+
+    // Add the data to the object
+    for (const [epoch, data] of Object.entries(epochsValidatorsData)) {
+      const epochNumber = parseInt(epoch);
+
+      if (!epochsTagsValidatorsData[epochNumber]) epochsTagsValidatorsData[epochNumber] = Object.create(null);
+
+      epochsTagsValidatorsData[epochNumber][tag] = data;
+    }
+  }
+
+  return epochsTagsValidatorsData;
+}

packages/brain/src/modules/apiServers/indexer/routes/epochs/types.ts

@@ -0,0 +1,22 @@
+import { Request } from "express";
+import { Tag } from "@stakingbrain/common"; // Assuming this is defined somewhere
+
+// The query parameters before they are parsed or validated
+// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-empty-object-type
+export type RequestReceived = Request<{}, any, any, QueryParamsReceived>;
+
+type QueryParamsReceived = {
+  start: string | number;
+  end: string | number;
+  tag?: Tag[] | Tag; // Can be an array or a single value before validation
+};
+
+// The query parameters after they are validated and parsed
+// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-empty-object-type
+export type RequestParsed = Request<{}, any, any, QueryParamsParsed>;
+
+type QueryParamsParsed = {
+  start: number;
+  end: number;
+  tag?: Tag[]; // After validation, tag should be an array
+};

packages/brain/src/modules/apiServers/indexer/routes/epochs/validation.ts

@@ -0,0 +1,49 @@
+import { Response, NextFunction } from "express";
+import { Tag, tags } from "@stakingbrain/common";
+import { RequestReceived } from "./types.js";
+
+// Validation middleware for query parameters
+export function validateQueryParams(req: RequestReceived, res: Response, next: NextFunction): void {
+  const { start, end, tag } = req.query;
+
+  // validate start and end
+  if (!start || !end) {
+    res.status(400).json({ message: "query parameters start and end must be provided" });
+    return;
+  }
+  if (typeof start !== "string" || typeof end !== "string") {
+    res.status(400).json({ message: "query parameter start and end must be of type string" });
+    return;
+  }
+
+  // parse start
+  req.query.start = parseInt(start);
+  req.query.end = parseInt(end);
+  // check that start is less or equal to end
+  if (req.query.start > req.query.end) {
+    res.status(400).json({ message: "query parameter start must be less than or equal to end" });
+    return;
+  }
+
+  // Validate tag
+  if (tag) {
+    // tag may be of type string or array of strings otherwise return 400
+    if (typeof tag !== "string" && !Array.isArray(tag)) {
+      res.status(400).json({ message: "tag must be a string or an array of strings" });
+    }
+
+    // if tag is a string, convert it to an array
+    const tagsArray = Array.isArray(tag) ? tag : [tag];
+    const invalidTag = tagsArray.find((t) => !tags.includes(t as Tag));
+
+    if (invalidTag) {
+      res.status(400).json({ message: `invalid tag received: ${invalidTag}. Allowed tags are ${tags.join(", ")}` });
+      return;
+    }
+
+    // If validation passed, update req.query.tag to ensure it is always an array for downstream middleware
+    req.query.tag = tagsArray;
+  }
+
+  next(); // Continue to the next middleware or route handler
+}

packages/brain/src/modules/apiServers/indexer/routes/index.ts

@@ -0,0 +1 @@
+export * from "./epochs/index.js";

packages/brain/src/modules/apiServers/indexer/startIndexerApi.ts

@@ -0,0 +1,30 @@
+import express from "express";
+import cors from "cors";
+import logger from "../../logger/index.js";
+import http from "node:http";
+import { params } from "../../../params.js";
+import { corsOptions } from "./config.js";
+import { createIndexerEpochsRouter } from "./routes/index.js";
+import { BrainDataBase } from "../../db/index.js";
+import { PostgresClient } from "../../apiClients/index.js";
+
+export function startIndexerApi({
+  brainDb,
+  postgresClient
+}: {
+  brainDb: BrainDataBase;
+  postgresClient: PostgresClient;
+}): http.Server {
+  const app = express();
+  app.use(express.json());
+  app.use(cors(corsOptions));
+
+  app.use(createIndexerEpochsRouter({ brainDb, postgresClient }));
+
+  const server = new http.Server(app);
+  server.listen(params.indexerPort, () => {
+    logger.info(`Indexer API listening on port ${params.indexerPort}`);
+  });
+
+  return server;
+}

packages/brain/src/modules/apiServers/ui/calls/fetchValidatorsPerformanceData.ts

@@ -1,6 +1,6 @@
 import { PostgresClient } from "../../../../modules/apiClients/index.js";
 import type { NumberOfDaysToQuery } from "../../../../modules/validatorsDataIngest/types.js";
-import type { EpochsValidatorsMap } from "../../../../modules/apiClients/postgres/types.js";
+import type { EpochsValidatorsData } from "../../../../modules/apiClients/postgres/types.js";
 import { fetchAndProcessValidatorsData } from "../../../validatorsDataIngest/index.js";
 
 export async function fetchValidatorsPerformanceData({
@@ -15,7 +15,7 @@ export async function fetchValidatorsPerformanceData({
   numberOfDaysToQuery?: NumberOfDaysToQuery;
   minGenesisTime: number;
   secondsPerSlot: number;
-}): Promise<EpochsValidatorsMap> {
+}): Promise<EpochsValidatorsData> {
   return await fetchAndProcessValidatorsData({
     validatorIndexes,
     postgresClient,

packages/brain/src/modules/apiServers/ui/calls/types.ts

@@ -7,7 +7,7 @@ import type {
   BeaconchainPoolVoluntaryExitsPostRequest,
   Web3signerPostResponse,
   Web3signerHealthcheckResponse,
-  EpochsValidatorsMap
+  EpochsValidatorsData
 } from "../../../apiClients/types.js";
 import { NumberOfDaysToQuery } from "../../../validatorsDataIngest/types.js";
 
@@ -29,7 +29,7 @@ export interface RpcMethods {
   fetchValidatorsPerformanceData: (
     validatorIndexes: string[],
     numberOfDaysToQuery?: NumberOfDaysToQuery
-  ) => Promise<EpochsValidatorsMap>;
+  ) => Promise<EpochsValidatorsData>;
 }
 
 export type ActionRequestOrigin = "ui" | "api";

packages/brain/src/modules/validatorsDataIngest/index.ts

@@ -1,5 +1,5 @@
 import { PostgresClient } from "../apiClients/index.js";
-import type { EpochsValidatorsMap } from "../apiClients/postgres/types.js";
+import type { EpochsValidatorsData } from "../apiClients/postgres/types.js";
 import logger from "../logger/index.js";
 import { getStartAndEndEpochs } from "./getStartAndEndEpochs.js";
 import { NumberOfDaysToQuery } from "./types.js";
@@ -26,7 +26,7 @@ export async function fetchAndProcessValidatorsData({
   minGenesisTime: number; // import from backend index
   secondsPerSlot: number; // immport from backend index
   numberOfDaysToQuery?: NumberOfDaysToQuery;
-}): Promise<EpochsValidatorsMap> {
+}): Promise<EpochsValidatorsData> {
   logger.info("Processing epochs data");
 
   // Get start timestamp and end timestamp

packages/brain/src/params.ts

@@ -9,6 +9,7 @@ export const params = {
   uiPort: 80,
   launchpadPort: 3000,
   brainPort: 5000,
+  indexerPort: 7000,
   defaultValidatorsMonitorUrl: "https://validators-proofs.dappnode.io",
   defaultProofsOfValidationCron: 24 * 60 * 60 * 1000 // 1 day in ms
 };