From 0c97fbcf442f8c74e5b8b05d396c12f7903500e0 Mon Sep 17 00:00:00 2001 From: sebastianburckhardt Date: Fri, 12 Apr 2024 11:12:25 -0700 Subject: [PATCH 1/4] filter duplicate client requests --- .../PartitionState/PrefetchState.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs index ec6038f6..3bc91c13 100644 --- a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs +++ b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs @@ -74,13 +74,19 @@ public override void Process(WaitRequestReceived waitRequestEvent, EffectTracker void ProcessClientRequestEventWithPrefetch(ClientRequestEventWithPrefetch clientRequestEvent, EffectTracker effects) { if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.Read) - { - this.Partition.Assert(!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString), "PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString)"); - - // Issue a read request that fetches the instance state. - // We have to buffer this request in the pending list so we can recover it. + { + if (!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString)) + { + // Issue a read request that fetches the instance state. + // We buffer this request in the pending list so we can recover it, and can filter duplicates + // (as long as the duplicates appear soon after the original) - this.PendingPrefetches.Add(clientRequestEvent.EventIdString, clientRequestEvent); + this.PendingPrefetches.Add(clientRequestEvent.EventIdString, clientRequestEvent); + } + else + { + return; // this is a duplicate. Ignore it. + } } else { From 3c47c2577b7e079fe3fae6466afa606ec4029682 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Mon, 15 Apr 2024 13:57:16 -0700 Subject: [PATCH 2/4] Update src/DurableTask.Netherite/PartitionState/PrefetchState.cs Co-authored-by: David Justo --- src/DurableTask.Netherite/PartitionState/PrefetchState.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs index 3bc91c13..9dbabcd5 100644 --- a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs +++ b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs @@ -75,6 +75,8 @@ void ProcessClientRequestEventWithPrefetch(ClientRequestEventWithPrefetch client { if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.Read) { + // It's possible for EventHubs to duplicate client-to-partition events. Therefore, we perform a best-effort + // de-duplication of EH messages. For more details, see: https://github.com/microsoft/durabletask-netherite/pull/379 if (!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString)) { // Issue a read request that fetches the instance state. From 76121c04376e770ffbb0fcc2e144cbea57903af1 Mon Sep 17 00:00:00 2001 From: sebastianburckhardt Date: Mon, 15 Apr 2024 14:59:23 -0700 Subject: [PATCH 3/4] fix whitespace --- src/DurableTask.Netherite/PartitionState/PrefetchState.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs index 9dbabcd5..a7c77322 100644 --- a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs +++ b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs @@ -78,7 +78,7 @@ void ProcessClientRequestEventWithPrefetch(ClientRequestEventWithPrefetch client // It's possible for EventHubs to duplicate client-to-partition events. Therefore, we perform a best-effort // de-duplication of EH messages. For more details, see: https://github.com/microsoft/durabletask-netherite/pull/379 if (!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString)) - { + { // Issue a read request that fetches the instance state. // We buffer this request in the pending list so we can recover it, and can filter duplicates // (as long as the duplicates appear soon after the original) From 795dd18a54781351317b5fafac82c600196a6f28 Mon Sep 17 00:00:00 2001 From: sebastianburckhardt Date: Mon, 15 Apr 2024 15:01:01 -0700 Subject: [PATCH 4/4] address PR feedback: add warning when event is dropped --- src/DurableTask.Netherite/PartitionState/PrefetchState.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs index a7c77322..b8bae815 100644 --- a/src/DurableTask.Netherite/PartitionState/PrefetchState.cs +++ b/src/DurableTask.Netherite/PartitionState/PrefetchState.cs @@ -87,7 +87,9 @@ void ProcessClientRequestEventWithPrefetch(ClientRequestEventWithPrefetch client } else { - return; // this is a duplicate. Ignore it. + // this is a duplicate. Ignore it. + effects.EventTraceHelper?.TraceEventProcessingWarning($"Dropped duplicate client request {clientRequestEvent} id={clientRequestEvent.EventIdString}"); + return; } } else