From a1a4bf448bd4888e4d6643f9736f2ea44799d43c Mon Sep 17 00:00:00 2001
From: Scott Bommarito <scottbom@microsoft.com>
Date: Mon, 26 Nov 2018 13:46:43 -0800
Subject: [PATCH] V3 Monitoring - Improve logging when queue message processing
 fails (#410)

---
 src/Ng/Jobs/MonitoringProcessorJob.cs         | 28 +++++++++++++++----
 .../Utility/LogEvents.cs                      |  2 ++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs
index c1f58d2de..167b50dd1 100644
--- a/src/Ng/Jobs/MonitoringProcessorJob.cs
+++ b/src/Ng/Jobs/MonitoringProcessorJob.cs
@@ -129,15 +129,31 @@ private async Task HandleQueueMessageAsync(
                 // We can remove the message from the queue because it was processed.
                 messageWasProcessed = true;
             }
-            catch (Exception e)
+            catch (Exception validationFailedToRunException)
             {
-                // Validations failed to run! Save this failed status to storage.
-                await SaveFailedPackageMonitoringStatusAsync(queuedContext, e, token);
-                // We can then remove the message from the queue because this failed status can be used to requeue the message.
-                messageWasProcessed = true;
+                try
+                {
+                    // Validations failed to run! Save this failed status to storage.
+                    await SaveFailedPackageMonitoringStatusAsync(queuedContext, validationFailedToRunException, token);
+                    // We can then remove the message from the queue because this failed status can be used to requeue the message.
+                    messageWasProcessed = true;
+                }
+                catch (Exception failedValidationSaveFailureException)
+                {
+                    // We failed to run validations and failed to save the failed validation!
+                    // We were not able to process this message. We need to log the exceptions so we can debug the issue.
+                    var aggregateException = new AggregateException(
+                        "Validations failed to run and saving unsuccessful validation failed!", 
+                        new[] { validationFailedToRunException, failedValidationSaveFailureException });
+
+                    Logger.LogCritical(
+                        NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageFatalFailure,
+                        aggregateException,
+                        "Failed to process queue message");
+                }
             }
 
-            // Note that if both validations fail and saving the failure status fail, we cannot remove the message from the queue.
+            // If we failed to run validations and failed to save the failed validation, we cannot remove the message from the queue.
             if (messageWasProcessed)
             {
                 await _queue.RemoveAsync(queueMessage, token);
diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs
index 5a4428b29..6eff3340f 100644
--- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs
+++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs
@@ -13,5 +13,7 @@ public static class LogEvents
 
         public static EventId StatusDeserializationFailure = new EventId(903, "Status deserialization failed!");
         public static EventId StatusDeserializationFatalFailure = new EventId(904, "Status deserialization failed, and was unable to parse id and version from filename!");
+
+        public static EventId QueueMessageFatalFailure = new EventId(905, "Failed to process queue message");
     }
 }