chore: bump libp2p deps (#6015)

* chore: bump libp2p deps * Add resource leak detection in mocha for e2e * Remove all retries from network tests * Remove the timeout from workflow actions * Uncomment a test * Update hooks to show time in the log * Add a custom mocha wrapper to detected leaked resources * Use mocha wrapper for beacon-node e2e tests * Add multiple exit to mocha run * Add mocha exit with failure count * Update the process exit strategy * Exit the process with signal * Exit the process with signal * Stop the beacon node once * Abort the process * Enable debug logging for e2e tests * Add unique name to each node for identification * Add unique name to each node for identification * Fix gossip test close * Fixing peer manager close handler * Update the process of finding active resources * Add log prefix to validators to debug * Add log prefix to validators to debug * Add try/catch for network core close * Wait for peer disconnect * Updat the exit strategy to ignore default resources * Add more logs to investigate * Await for disconnect * Add debug level logs threads * Remove exit option for the mocha * Cleanup extra code * Cleanup the PR * Cleanup a duplicate variable * Fix the unused tests --------- Co-authored-by: Nazar Hussain <[email protected]>
ChainSafe · Oct 13, 2023 · ab2dfdd · ab2dfdd
1 parent c6291ad
commit ab2dfdd
Show file tree

Hide file tree

Showing 22 changed files with 281 additions and 285 deletions.
diff --git a/package.json b/package.json
@@ -75,7 +75,7 @@
     "karma-spec-reporter": "^0.0.36",
     "karma-webpack": "^5.0.0",
     "lerna": "^7.3.0",
-    "libp2p": "0.46.3",
+    "libp2p": "0.46.12",
     "mocha": "^10.2.0",
     "node-gyp": "^9.4.0",
     "npm-run-all": "^4.1.5",

diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json
@@ -101,7 +101,7 @@
     "@chainsafe/blst": "^0.2.9",
     "@chainsafe/discv5": "^5.1.0",
     "@chainsafe/libp2p-gossipsub": "^10.1.0",
-    "@chainsafe/libp2p-noise": "^13.0.0",
+    "@chainsafe/libp2p-noise": "^13.0.1",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/ssz": "^0.13.0",
@@ -111,14 +111,14 @@
     "@fastify/cors": "^8.2.1",
     "@fastify/swagger": "^8.10.0",
     "@fastify/swagger-ui": "^1.9.3",
-    "@libp2p/bootstrap": "^9.0.2",
-    "@libp2p/interface": "^0.1.1",
-    "@libp2p/mdns": "^9.0.2",
-    "@libp2p/mplex": "^9.0.2",
-    "@libp2p/peer-id": "^3.0.1",
-    "@libp2p/peer-id-factory": "^3.0.2",
-    "@libp2p/prometheus-metrics": "^2.0.2",
-    "@libp2p/tcp": "8.0.2",
+    "@libp2p/bootstrap": "^9.0.7",
+    "@libp2p/interface": "^0.1.2",
+    "@libp2p/mdns": "^9.0.9",
+    "@libp2p/mplex": "^9.0.7",
+    "@libp2p/peer-id": "^3.0.2",
+    "@libp2p/peer-id-factory": "^3.0.4",
+    "@libp2p/prometheus-metrics": "^2.0.7",
+    "@libp2p/tcp": "8.0.8",
     "@lodestar/api": "^1.11.3",
     "@lodestar/config": "^1.11.3",
     "@lodestar/db": "^1.11.3",
@@ -143,7 +143,7 @@
     "it-all": "^3.0.2",
     "it-pipe": "^3.0.1",
     "jwt-simple": "0.5.6",
-    "libp2p": "0.46.3",
+    "libp2p": "0.46.12",
     "multiformats": "^11.0.1",
     "prom-client": "^14.2.0",
     "qs": "^6.11.1",

diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts
@@ -148,6 +148,7 @@ export class WorkerNetworkCore implements INetworkCore {
   }
 
   async close(): Promise<void> {
+    this.modules.logger.debug("closing network core running in network worker");
     await this.getApi().close();
     this.modules.logger.debug("terminating network worker");
     await terminateWorkerThread({

diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts
@@ -660,7 +660,7 @@ export class PeerManager {
     } catch (e) {
       this.logger.verbose("Failed to send goodbye", {peer: prettyPrintPeerId(peer)}, e as Error);
     } finally {
-      void this.disconnect(peer);
+      await this.disconnect(peer);
     }
   }
 

diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts
@@ -93,6 +93,7 @@ describe("beacon node api", function () {
 
       // To make BN communicate with EL, it needs to produce some blocks and for that need validators
       const {validators} = await getAndInitDevValidators({
+        logPrefix: "Offline-BN",
         node: bnElOffline,
         validatorClientCount: 1,
         validatorsPerClient: validatorCount,

diff --git a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts
@@ -24,7 +24,7 @@ describe("lightclient api", function () {
   const genesisValidatorsRoot = Buffer.alloc(32, 0xaa);
   const config = createBeaconConfig(chainConfig, genesisValidatorsRoot);
   const testLoggerOpts: TestLoggerOpts = {level: LogLevel.info};
-  const loggerNodeA = testLogger("Node-A", testLoggerOpts);
+  const loggerNodeA = testLogger("lightclient-api", testLoggerOpts);
   const validatorCount = 2;
 
   let bn: BeaconNode;
@@ -54,6 +54,7 @@ describe("lightclient api", function () {
     validators = (
       await getAndInitDevValidators({
         node: bn,
+        logPrefix: "lightclient-api",
         validatorsPerClient: validatorCount,
         validatorClientCount: 1,
         startIndex: 0,

diff --git a/packages/beacon-node/test/e2e/chain/lightclient.test.ts b/packages/beacon-node/test/e2e/chain/lightclient.test.ts
@@ -67,7 +67,7 @@ describe("chain / lightclient", function () {
       },
     };
 
-    const loggerNodeA = testLogger("Node", testLoggerOpts);
+    const loggerNodeA = testLogger("lightclientNode", testLoggerOpts);
     const loggerLC = testLogger("LC", {...testLoggerOpts, level: LogLevel.debug});
 
     const bn = await getDevBeaconNode({
@@ -89,6 +89,7 @@ describe("chain / lightclient", function () {
 
     const {validators} = await getAndInitDevValidators({
       node: bn,
+      logPrefix: "lightclientNode",
       validatorsPerClient: validatorCount,
       validatorClientCount,
       startIndex: 0,

diff --git a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts
@@ -47,7 +47,7 @@ describe.skip("doppelganger / doppelganger test", function () {
 
   async function createBNAndVC(config?: TestConfig): Promise<{beaconNode: BeaconNode; validators: Validator[]}> {
     const testLoggerOpts: TestLoggerOpts = {level: LogLevel.info};
-    const loggerNodeA = testLogger("Node-A", testLoggerOpts);
+    const loggerNodeA = testLogger("doppelganger", testLoggerOpts);
 
     const bn = await getDevBeaconNode({
       params: beaconParams,
@@ -65,6 +65,7 @@ describe.skip("doppelganger / doppelganger test", function () {
 
     const {validators: validatorsWithDoppelganger} = await getAndInitDevValidators({
       node: bn,
+      logPrefix: "doppelganger",
       validatorsPerClient: validatorCount,
       validatorClientCount: 1,
       startIndex: 0,
@@ -161,6 +162,7 @@ describe.skip("doppelganger / doppelganger test", function () {
     });
 
     const {validators: validator0WithoutDoppelganger} = await getAndInitDevValidators({
+      logPrefix: "doppelganger2",
       node: bn,
       validatorsPerClient: validatorCount,
       validatorClientCount: 1,

diff --git a/packages/beacon-node/test/e2e/network/gossipsub.test.ts b/packages/beacon-node/test/e2e/network/gossipsub.test.ts
@@ -19,7 +19,6 @@ describe("gossipsub / worker", function () {
 
 function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void {
   if (this.timeout() < 20 * 1000) this.timeout(150 * 1000);
-  this.retries(2); // This test fail sometimes, with a 5% rate.
 
   const afterEachCallbacks: (() => Promise<void> | void)[] = [];
   afterEach(async () => {
@@ -41,8 +40,14 @@ function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void {
 
   // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
   async function mockModules(gossipHandlersPartial?: Partial<GossipHandlers>) {
-    const [netA, closeA] = await getNetworkForTest("A", config, {opts: {useWorker}, gossipHandlersPartial});
-    const [netB, closeB] = await getNetworkForTest("B", config, {opts: {useWorker}, gossipHandlersPartial});
+    const [netA, closeA] = await getNetworkForTest(`gossipsub-${useWorker ? "worker" : "main"}-A`, config, {
+      opts: {useWorker},
+      gossipHandlersPartial,
+    });
+    const [netB, closeB] = await getNetworkForTest(`gossipsub-${useWorker ? "worker" : "main"}-B`, config, {
+      opts: {useWorker},
+      gossipHandlersPartial,
+    });
 
     afterEachCallbacks.push(async () => {
       await closeA();

diff --git a/packages/beacon-node/test/e2e/network/mdns.test.ts b/packages/beacon-node/test/e2e/network/mdns.test.ts
@@ -28,7 +28,6 @@ const mu = "/ip4/127.0.0.1/tcp/0";
 // eslint-disable-next-line mocha/no-skipped-tests
 describe.skip("mdns", function () {
   this.timeout(50000);
-  this.retries(2); // This test fail sometimes, with a 5% rate.
 
   const afterEachCallbacks: (() => Promise<void> | void)[] = [];
   afterEach(async () => {
@@ -121,7 +120,7 @@ describe.skip("mdns", function () {
 
   // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
   async function createTestNodesAB() {
-    return Promise.all([createTestNode("A"), createTestNode("B")]);
+    return Promise.all([createTestNode("mdns-A"), createTestNode("mdns-B")]);
   }
 
   it("should connect two peers on a LAN", async function () {

diff --git a/packages/beacon-node/test/e2e/network/network.test.ts b/packages/beacon-node/test/e2e/network/network.test.ts
@@ -21,42 +21,47 @@ describe("network / worker", function () {
 
 function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void {
   this.timeout(50000);
-  this.retries(2); // This test fail sometimes, with a 5% rate.
 
   const afterEachCallbacks: (() => Promise<void> | void)[] = [];
   afterEach(async () => {
-    await Promise.all(afterEachCallbacks.map((cb) => cb()));
-    afterEachCallbacks.splice(0, afterEachCallbacks.length);
+    while (afterEachCallbacks.length > 0) {
+      const callback = afterEachCallbacks.pop();
+      if (callback) await callback();
+    }
   });
 
   let controller: AbortController;
   beforeEach(() => (controller = new AbortController()));
-  afterEach(() => controller.abort());
+  afterEach(() => {
+    controller.abort();
+    sinon.restore();
+  });
 
   // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
   async function createTestNode(nodeName: string) {
     const [network, closeAll] = await getNetworkForTest(nodeName, config, {opts: {useWorker}});
 
     afterEachCallbacks.push(async () => {
       await closeAll();
-      sinon.restore();
     });
 
     return network;
   }
 
-  // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-  async function createTestNodesAB() {
-    return Promise.all([createTestNode("A"), createTestNode("B")]);
+  async function createTestNodesAB(): Promise<[Network, Network]> {
+    return Promise.all([
+      createTestNode(`network-${useWorker ? "worker" : "main"}-A`),
+      createTestNode(`network-${useWorker ? "worker" : "main"}-A`),
+    ]);
   }
 
   it("Disconnect peer", async () => {
-    const network = await createTestNode("A");
+    const network = await createTestNode(`network-${useWorker ? "worker" : "main"}-DP`);
     await network.disconnectPeer(getValidPeerId().toString());
   });
 
   it("return getNetworkIdentity", async () => {
-    const network = await createTestNode("A");
+    const network = await createTestNode(`network-${useWorker ? "worker" : "main"}-NI`);
     const networkIdentity = await network.getNetworkIdentity();
     expect(networkIdentity.peerId).equals(network.peerId.toString());
   });
@@ -125,7 +130,7 @@ function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void {
   });
 
   it("Should subscribe to gossip core topics on demand", async () => {
-    const netA = await createTestNode("A");
+    const netA = await createTestNode(`network-${useWorker ? "worker" : "main"}-CT`);
 
     expect(await getTopics(netA)).deep.equals([]);
 

diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts
@@ -20,7 +20,7 @@ import {IAttnetsService} from "../../../../src/network/subnets/index.js";
 import {Clock} from "../../../../src/util/clock.js";
 import {LocalStatusCache} from "../../../../src/network/statusCache.js";
 
-const logger = testLogger();
+const logger = testLogger("peerManager");
 
 describe("network / peers / PeerManager", function () {
   const peerId1 = getValidPeerId();
@@ -93,6 +93,10 @@ describe("network / peers / PeerManager", function () {
       null
     );
 
+    afterEachCallbacks.push(async () => {
+      await peerManager.close();
+    });
+
     return {statusCache, clock, libp2p, reqResp, peerManager, networkEventBus};
   }
 

diff --git a/packages/beacon-node/test/e2e/network/reqresp.test.ts b/packages/beacon-node/test/e2e/network/reqresp.test.ts
@@ -29,7 +29,6 @@ describe("network / reqresp / worker", function () {
 
 function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void {
   if (this.timeout() < 60_000) this.timeout(60_000);
-  this.retries(2); // This test fail sometimes, with a 5% rate.
 
   // Schedule ALTAIR_FORK_EPOCH to trigger registering lightclient ReqResp protocols immediately
   const config = createChainForkConfig({
@@ -56,8 +55,14 @@ function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void {
     getReqRespHandler?: GetReqRespHandlerFn,
     opts?: ReqRespBeaconNodeOpts
   ): Promise<[Network, Network, PeerIdStr, PeerIdStr]> {
-    const [netA, closeA] = await getNetworkForTest("A", config, {getReqRespHandler, opts: {...opts, useWorker}});
-    const [netB, closeB] = await getNetworkForTest("B", config, {getReqRespHandler, opts: {...opts, useWorker}});
+    const [netA, closeA] = await getNetworkForTest(`reqresp-${useWorker ? "worker" : "main"}-A`, config, {
+      getReqRespHandler,
+      opts: {...opts, useWorker},
+    });
+    const [netB, closeB] = await getNetworkForTest(`reqresp-${useWorker ? "worker" : "main"}-B`, config, {
+      getReqRespHandler,
+      opts: {...opts, useWorker},
+    });
 
     afterEachCallbacks.push(async () => {
       await closeA();

diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts
@@ -59,8 +59,8 @@ describe("sync / unknown block sync", function () {
         },
       };
 
-      const loggerNodeA = testLogger("Node-A", testLoggerOpts);
-      const loggerNodeB = testLogger("Node-B", testLoggerOpts);
+      const loggerNodeA = testLogger("UnknownSync-Node-A", testLoggerOpts);
+      const loggerNodeB = testLogger("UnknownSync-Node-B", testLoggerOpts);
 
       const bn = await getDevBeaconNode({
         params: testParams,
@@ -73,10 +73,9 @@ describe("sync / unknown block sync", function () {
         logger: loggerNodeA,
       });
 
-      afterEachCallbacks.push(() => bn.close());
-
       const {validators} = await getAndInitDevValidators({
         node: bn,
+        logPrefix: "UnknownSync",
         validatorsPerClient: validatorCount,
         validatorClientCount: 1,
         startIndex: 0,

diff --git a/packages/beacon-node/test/sim/4844-interop.test.ts b/packages/beacon-node/test/sim/4844-interop.test.ts
@@ -171,6 +171,7 @@ describe("executionEngine / ExecutionEngineHttp", function () {
     const {data: bnIdentity} = await bn.api.node.getNetworkIdentity();
 
     const {validators} = await getAndInitDevValidators({
+      logPrefix: "Node-A",
       node: bn,
       validatorsPerClient,
       validatorClientCount,

diff --git a/packages/beacon-node/test/sim/merge-interop.test.ts b/packages/beacon-node/test/sim/merge-interop.test.ts
@@ -339,6 +339,7 @@ describe("executionEngine / ExecutionEngineHttp", function () {
     } as ValidatorProposerConfig;
 
     const {validators} = await getAndInitDevValidators({
+      logPrefix: "Node-A",
       node: bn,
       validatorsPerClient,
       validatorClientCount,

diff --git a/packages/beacon-node/test/sim/mergemock.test.ts b/packages/beacon-node/test/sim/mergemock.test.ts
@@ -192,6 +192,7 @@ describe("executionEngine / ExecutionEngineHttp", function () {
     } as ValidatorProposerConfig;
 
     const {validators} = await getAndInitDevValidators({
+      logPrefix: "mergemock",
       node: bn,
       validatorsPerClient,
       validatorClientCount,

diff --git a/packages/beacon-node/test/sim/withdrawal-interop.test.ts b/packages/beacon-node/test/sim/withdrawal-interop.test.ts
@@ -285,6 +285,7 @@ describe("executionEngine / ExecutionEngineHttp", function () {
     } as ValidatorProposerConfig;
 
     const {validators} = await getAndInitDevValidators({
+      logPrefix: "withdrawal-interop",
       node: bn,
       validatorsPerClient,
       validatorClientCount,

diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts
@@ -10,6 +10,7 @@ import {testLogger, TestLoggerOpts} from "../logger.js";
 
 export async function getAndInitDevValidators({
   node,
+  logPrefix,
   validatorsPerClient = 8,
   validatorClientCount = 1,
   startIndex = 0,
@@ -20,6 +21,7 @@ export async function getAndInitDevValidators({
   valProposerConfig,
 }: {
   node: BeaconNode;
+  logPrefix: string;
   validatorsPerClient: number;
   validatorClientCount: number;
   startIndex: number;
@@ -36,7 +38,7 @@ export async function getAndInitDevValidators({
   for (let clientIndex = 0; clientIndex < validatorClientCount; clientIndex++) {
     const startIndexVc = startIndex + clientIndex * validatorsPerClient;
     const endIndex = startIndexVc + validatorsPerClient - 1;
-    const logger = testLogger(`Vali ${startIndexVc}-${endIndex}`, testLoggerOpts);
+    const logger = testLogger(`${logPrefix}-VAL-${startIndexVc}-${endIndex}`, testLoggerOpts);
     const tmpDir = tmp.dirSync({unsafeCleanup: true});
     const db = await LevelDbController.create({name: tmpDir.name}, {logger});
     const slashingProtection = new SlashingProtection(db);

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -61,10 +61,10 @@
     "@chainsafe/discv5": "^5.1.0",
     "@chainsafe/ssz": "^0.13.0",
     "@chainsafe/threads": "^1.11.1",
+    "@libp2p/crypto": "^2.0.4",
+    "@libp2p/peer-id": "^3.0.2",
+    "@libp2p/peer-id-factory": "^3.0.4",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
-    "@libp2p/crypto": "^2.0.2",
-    "@libp2p/peer-id": "^3.0.1",
-    "@libp2p/peer-id-factory": "^3.0.2",
     "@lodestar/api": "^1.11.3",
     "@lodestar/beacon-node": "^1.11.3",
     "@lodestar/config": "^1.11.3",