Add debug level logs and sync test to sim tests (#4673)

* Update sim tests structure to add new nodes

* Add new node to work to test sync

* Add sim test for checkpoint sync

* Update the checkpoint sync root

* Update seconds per slot for the simulation tests

* Update the assertion for attestaion count

* Update expectedMinParticipationRate and expectedMinSyncParticipationRate to 90%

* Update genesis delay

* Increase genesisSlotsDelay for ci

* Increase genesisSlotsDelay for ci

* Update test timeout

* Update the code with the feedback

* Disable file logging for the lodestar node during sim tests

packages/cli/test/simulation/simulation.test.ts

@@ -1,7 +1,8 @@
 import {join} from "node:path";
 import {Epoch} from "@lodestar/types";
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {logFilesDir, SimulationEnvironment} from "../utils/simulation/index.js";
+import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {logFilesDir} from "../utils/simulation/utils.js";
 import {
   missedBlocksAssertions,
   attestationParticipationAssertions,
@@ -12,6 +13,7 @@ import {
   headsAssertions,
   syncCommitteeAssertions,
 } from "../utils/simulation/assertions.js";
+import {CLClient, CLParticipant, Job} from "../utils/simulation/interfaces.js";
 
 const nodeCases: {beaconNodes: number; validatorClients: number; validatorsPerClient: number}[] = [
   {beaconNodes: 4, validatorClients: 1, validatorsPerClient: 32},
@@ -31,8 +33,6 @@ const forksCases: {
   },
 ];
 
-let testCases = 0;
-
 for (const {beaconNodes, validatorClients, validatorsPerClient} of nodeCases) {
   for (const {
     title,
@@ -62,14 +62,13 @@ for (const {beaconNodes, validatorClients, validatorsPerClient} of nodeCases) {
       validatorsPerClient,
       altairEpoch,
       // TODO: Use extra delay until env.clock is based on absolute time
-      genesisSlotsDelay: (SLOTS_PER_EPOCH * runTill + 50) * testCases + 30,
+      genesisSlotsDelay: SLOTS_PER_EPOCH * 2,
       bellatrixEpoch,
       logFilesDir: join(logFilesDir, testIdStr),
     });
-    testCases += 1;
 
     describe(`simulation test - ${testIdStr}`, function () {
-      this.timeout("5m");
+      this.timeout("10m");
 
       describe(title, () => {
         before("start env", async () => {
@@ -123,6 +122,91 @@ for (const {beaconNodes, validatorClients, validatorsPerClient} of nodeCases) {
             });
           });
         }
+
+        describe("range sync from genesis", () => {
+          let clParticipant: CLParticipant;
+          let clJob: Job;
+          const rangeSyncEpoch = runTill + 1;
+
+          before(async () => {
+            const {job, participant} = env.createCLClient(CLClient.Lodestar, env.nodes.length, {
+              id: "range-sync-node",
+              secretKeys: [],
+            });
+            clJob = job;
+            clParticipant = participant;
+
+            await clJob.start();
+            await env.network.connectNewNode(clParticipant);
+
+            env.tracker.track(clParticipant);
+            // Wait for existing tests to finish
+            await env.waitForSlot(env.clock.getLastSlotOfEpoch(rangeSyncEpoch) + 1);
+
+            env.tracker.printNoesInfo(rangeSyncEpoch);
+          });
+
+          after(async () => {
+            await clJob.stop();
+          });
+
+          describe("missed blocks", () => {
+            missedBlocksAssertions(env, rangeSyncEpoch);
+          });
+
+          describe("finality", () => {
+            finalityAssertions(env, rangeSyncEpoch);
+          });
+
+          describe("heads", () => {
+            headsAssertions(env, rangeSyncEpoch);
+          });
+        });
+
+        describe("checkpoint sync", () => {
+          let clParticipant: CLParticipant;
+          let clJob: Job;
+          const checkpointSyncEpoch = runTill + 2;
+
+          before(async () => {
+            const {
+              data: {finalized},
+            } = await env.nodes[0].api.beacon.getStateFinalityCheckpoints("head");
+
+            const {job, participant} = env.createCLClient(CLClient.Lodestar, env.nodes.length, {
+              id: "checkpoint-sync-node",
+              secretKeys: [],
+              wssCheckpoint: `${finalized.root}:${finalized.epoch}`,
+            });
+            clJob = job;
+            clParticipant = participant;
+
+            await clJob.start();
+            await env.network.connectNewNode(clParticipant);
+
+            env.tracker.track(clParticipant);
+            // Wait for existing tests to finish
+            await env.waitForSlot(env.clock.getLastSlotOfEpoch(checkpointSyncEpoch) + 1);
+
+            env.tracker.printNoesInfo(checkpointSyncEpoch);
+          });
+
+          after(async () => {
+            await clJob.stop();
+          });
+
+          describe("missed blocks", () => {
+            missedBlocksAssertions(env, checkpointSyncEpoch);
+          });
+
+          describe("finality", () => {
+            finalityAssertions(env, checkpointSyncEpoch);
+          });
+
+          describe("heads", () => {
+            headsAssertions(env, checkpointSyncEpoch);
+          });
+        });
       });
     });
   }

packages/cli/test/utils/simulation/ExternalSignerServer.ts

@@ -12,13 +12,10 @@ export class ExternalSignerServer {
   readonly port: number;
 
   private server: ReturnType<typeof fastify>;
+  private secretKeyMap = new Map<string, SecretKey>();
 
   constructor(secretKeys: SecretKey[]) {
-    const secretKeyMap = new Map<string, SecretKey>();
-    for (const secretKey of secretKeys) {
-      const pubkeyHex = secretKey.toPublicKey().toHex();
-      secretKeyMap.set(pubkeyHex, secretKey);
-    }
+    this.addKeys(secretKeys);
     ExternalSignerServer.totalProcessCount++;
     this.port = EXTERNAL_SIGNER_BASE_PORT + ExternalSignerServer.totalProcessCount;
 
@@ -29,7 +26,7 @@ export class ExternalSignerServer {
     });
 
     this.server.get("/api/v1/eth2/publicKeys", async () => {
-      return [...secretKeyMap.keys()];
+      return [...this.secretKeyMap.keys()];
     });
 
     /* eslint-disable @typescript-eslint/naming-convention */
@@ -46,7 +43,7 @@ export class ExternalSignerServer {
       const pubkeyHex: string = req.params.identifier;
       const signingRootHex: string = req.body.signingRoot;
 
-      const secretKey = secretKeyMap.get(pubkeyHex);
+      const secretKey = this.secretKeyMap.get(pubkeyHex);
       if (!secretKey) {
         throw Error(`pubkey not known ${pubkeyHex}`);
       }
@@ -55,19 +52,24 @@ export class ExternalSignerServer {
     });
   }
 
+  addKeys(secretKeys: SecretKey[]): void {
+    for (const secretKey of secretKeys) {
+      const pubkeyHex = secretKey.toPublicKey().toHex();
+      this.secretKeyMap.set(pubkeyHex, secretKey);
+    }
+  }
+
   get url(): string {
     return `http://${this.address}:${this.port}`;
   }
 
   async start(): Promise<void> {
     console.log(`Starting external signer server at ${this.url}.`);
     await this.server.listen(this.port, this.address);
-    console.log(`Started external signer server at ${this.url}.`);
   }
 
   async stop(): Promise<void> {
     console.log(`Stopping external signer server at ${this.url}.`);
     await this.server.close();
-    console.log(`Stopped external signer server at ${this.url}.`);
   }
 }