From 460119d0bc77badb5c8fad0b43ea56cfea9b67a0 Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 15:45:55 -0300 Subject: [PATCH 1/8] Do not mark failed to recover as seen --- relayer/middleware/missedVaasV3/check.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/relayer/middleware/missedVaasV3/check.ts b/relayer/middleware/missedVaasV3/check.ts index dc4b4ba2..78f06a50 100644 --- a/relayer/middleware/missedVaasV3/check.ts +++ b/relayer/middleware/missedVaasV3/check.ts @@ -171,15 +171,13 @@ export async function checkForMissedVaas( failedToRecover, ); - const allSeenVaas = processed.concat(failedToRecover); - - if (allSeenVaas.length) + if (processed.length) await batchMarkAsSeen( redis, prefix, emitterChain, emitterAddress, - allSeenVaas, + processed, ); return { @@ -338,6 +336,8 @@ async function lookAhead( lastVisitedSequence = vaa.sequence; } + + return { lookAheadSequences, processed, failedToRecover }; } From a641acc6962fe84b4fdd5fb06772c65c89dab262 Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 16:24:53 -0300 Subject: [PATCH 2/8] Fix bigint mixed types issue --- relayer/middleware/missedVaasV3/check.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relayer/middleware/missedVaasV3/check.ts b/relayer/middleware/missedVaasV3/check.ts index 78f06a50..e1da945b 100644 --- a/relayer/middleware/missedVaasV3/check.ts +++ b/relayer/middleware/missedVaasV3/check.ts @@ -309,11 +309,11 @@ async function lookAhead( }`, ); - let lastVisitedSequence: bigint = lastSeenSequence; + let lastVisitedSequence: bigint = BigInt(lastSeenSequence); for (const vaa of vaas) { lookAheadSequences.push(vaa.sequence.toString()); - const sequenceGap = BigInt(vaa.sequence) - BigInt(lastVisitedSequence); + const sequenceGap = BigInt(vaa.sequence) - lastVisitedSequence; if (sequenceGap > 0) { const missing = Array.from({ length: Number(sequenceGap - 1n) }, (_, i) => (lastVisitedSequence + BigInt(i + 1)).toString(), @@ -333,7 +333,7 @@ async function lookAhead( ); } - lastVisitedSequence = vaa.sequence; + lastVisitedSequence = BigInt(vaa.sequence); } From fec06c4831f6e7092a29c16747beb72ecfea8a7d Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 16:43:33 -0300 Subject: [PATCH 3/8] Include last safe sequence in the seen sequenes array --- relayer/middleware/missedVaasV3/check.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/relayer/middleware/missedVaasV3/check.ts b/relayer/middleware/missedVaasV3/check.ts index e1da945b..6805dbbc 100644 --- a/relayer/middleware/missedVaasV3/check.ts +++ b/relayer/middleware/missedVaasV3/check.ts @@ -61,13 +61,16 @@ export async function checkForMissedVaas( let missingSequences: bigint[] = []; if (seenSequences.length) { - const first = seenSequences[0]; + const first = (previousSafeSequence !== null && previousSafeSequence < seenSequences[0]) ? previousSafeSequence : seenSequences[0]; const last = seenSequences[seenSequences.length - 1]; logger?.info( `Scanning sequences from ${first} to ${last} for missing sequences`, ); // Check if there is any leap between the sequences seen, // and try reprocessing them if any: + if (previousSafeSequence !== null && previousSafeSequence < seenSequences[0]) { + seenSequences.unshift(previousSafeSequence); + } missingSequences = scanForSequenceLeaps(seenSequences); await mapConcurrent( From 02c1b9c161f191647c689779be2afb253167c7f2 Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 17:04:38 -0300 Subject: [PATCH 4/8] Remove processed from failedToRecover --- relayer/middleware/missedVaasV3/storage.ts | 31 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/relayer/middleware/missedVaasV3/storage.ts b/relayer/middleware/missedVaasV3/storage.ts index 937d0e68..c8cb75ba 100644 --- a/relayer/middleware/missedVaasV3/storage.ts +++ b/relayer/middleware/missedVaasV3/storage.ts @@ -22,11 +22,18 @@ export function batchMarkAsSeen( emitterAddress: string, sequences: string[], ) { - return batchAddToSet( - redis, - getSeenVaaKey(prefix, emitterChain, emitterAddress), - sequences, - ); + return Promise.all([ + batchAddToSet( + redis, + getSeenVaaKey(prefix, emitterChain, emitterAddress), + sequences, + ), + batchRemoveFromSet( + redis, + getFailedToFetchKey(prefix, emitterChain, emitterAddress), + sequences, + ) + ]); } export function batchMarkAsFailedToRecover( @@ -331,3 +338,17 @@ async function batchAddToSet( await pipeline.exec(); } + +async function batchRemoveFromSet( + redis: Cluster | Redis, + key: string, + sequences: string[], +) { + const pipeline = redis.pipeline(); + + for (const sequence of sequences) { + pipeline.zrem(key, sequence); + } + + return pipeline.exec(); +} \ No newline at end of file From fce83fd2d0f770723ed2eeedf7d4b0afb3ccc7d3 Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 17:30:25 -0300 Subject: [PATCH 5/8] Run prettier --- relayer/middleware/missedVaasV3/check.ts | 12 ++++++++---- relayer/middleware/missedVaasV3/storage.ts | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/relayer/middleware/missedVaasV3/check.ts b/relayer/middleware/missedVaasV3/check.ts index 6805dbbc..866ae667 100644 --- a/relayer/middleware/missedVaasV3/check.ts +++ b/relayer/middleware/missedVaasV3/check.ts @@ -61,14 +61,20 @@ export async function checkForMissedVaas( let missingSequences: bigint[] = []; if (seenSequences.length) { - const first = (previousSafeSequence !== null && previousSafeSequence < seenSequences[0]) ? previousSafeSequence : seenSequences[0]; + const first = + previousSafeSequence !== null && previousSafeSequence < seenSequences[0] + ? previousSafeSequence + : seenSequences[0]; const last = seenSequences[seenSequences.length - 1]; logger?.info( `Scanning sequences from ${first} to ${last} for missing sequences`, ); // Check if there is any leap between the sequences seen, // and try reprocessing them if any: - if (previousSafeSequence !== null && previousSafeSequence < seenSequences[0]) { + if ( + previousSafeSequence !== null && + previousSafeSequence < seenSequences[0] + ) { seenSequences.unshift(previousSafeSequence); } missingSequences = scanForSequenceLeaps(seenSequences); @@ -339,8 +345,6 @@ async function lookAhead( lastVisitedSequence = BigInt(vaa.sequence); } - - return { lookAheadSequences, processed, failedToRecover }; } diff --git a/relayer/middleware/missedVaasV3/storage.ts b/relayer/middleware/missedVaasV3/storage.ts index c8cb75ba..cd625e95 100644 --- a/relayer/middleware/missedVaasV3/storage.ts +++ b/relayer/middleware/missedVaasV3/storage.ts @@ -32,7 +32,7 @@ export function batchMarkAsSeen( redis, getFailedToFetchKey(prefix, emitterChain, emitterAddress), sequences, - ) + ), ]); } @@ -351,4 +351,4 @@ async function batchRemoveFromSet( } return pipeline.exec(); -} \ No newline at end of file +} From 4f91dc7a357268f0383b0b43aabceb090fb3f273 Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 17:38:31 -0300 Subject: [PATCH 6/8] Manue prettier fix --- examples/simple/tsconfig.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/simple/tsconfig.json b/examples/simple/tsconfig.json index 3c27ba12..115f82bd 100644 --- a/examples/simple/tsconfig.json +++ b/examples/simple/tsconfig.json @@ -18,7 +18,5 @@ "allowSyntheticDefaultImports": true }, "include": ["src"], - "exclude": [ - "node_modules", - ] + "exclude": ["node_modules"] } From 125d9693e01dc94932ca52391c5195f3761da87d Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 17:57:12 -0300 Subject: [PATCH 7/8] Fix missed vaa tests --- test/middleware/missedVaasV3/check.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/middleware/missedVaasV3/check.test.ts b/test/middleware/missedVaasV3/check.test.ts index 2a001e8a..898d992f 100644 --- a/test/middleware/missedVaasV3/check.test.ts +++ b/test/middleware/missedVaasV3/check.test.ts @@ -225,8 +225,8 @@ describe("MissedVaaV3.check", () => { const markedSeen = batchMarkAsSeenMock.mock.calls[0][4]; - // VAAs marked as failed to recover are marked as seen - expect(markedSeen.length).toEqual(2); + // VAAs marked as failed to recover are NOT marked as seen + expect(markedSeen.length).toEqual(1); }); test("If a sequence fails to be reprocessed it won't be marked as seen and won't interrupt the processing of other seqs", async () => { From d542922c76028032a9d86d63811fb313cbe157b6 Mon Sep 17 00:00:00 2001 From: Cristian Iturri Date: Thu, 26 Oct 2023 18:00:21 -0300 Subject: [PATCH 8/8] Update relayer/middleware/missedVaasV3/check.ts Co-authored-by: scnale --- relayer/middleware/missedVaasV3/check.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relayer/middleware/missedVaasV3/check.ts b/relayer/middleware/missedVaasV3/check.ts index 866ae667..d0db9dcd 100644 --- a/relayer/middleware/missedVaasV3/check.ts +++ b/relayer/middleware/missedVaasV3/check.ts @@ -318,7 +318,7 @@ async function lookAhead( }`, ); - let lastVisitedSequence: bigint = BigInt(lastSeenSequence); + let lastVisitedSequence = BigInt(lastSeenSequence); for (const vaa of vaas) { lookAheadSequences.push(vaa.sequence.toString());