V3 Monitoring - Improve logging when queue message processing fails (#…

…410)
NuGet · Nov 26, 2018 · a1a4bf4 · a1a4bf4
1 parent 49ed106
commit a1a4bf4
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 6 deletions.
diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs
@@ -129,15 +129,31 @@ private async Task HandleQueueMessageAsync(
                 // We can remove the message from the queue because it was processed.
                 messageWasProcessed = true;
             }
-            catch (Exception e)
+            catch (Exception validationFailedToRunException)
             {
-                // Validations failed to run! Save this failed status to storage.
-                await SaveFailedPackageMonitoringStatusAsync(queuedContext, e, token);
-                // We can then remove the message from the queue because this failed status can be used to requeue the message.
-                messageWasProcessed = true;
+                try
+                {
+                    // Validations failed to run! Save this failed status to storage.
+                    await SaveFailedPackageMonitoringStatusAsync(queuedContext, validationFailedToRunException, token);
+                    // We can then remove the message from the queue because this failed status can be used to requeue the message.
+                    messageWasProcessed = true;
+                }
+                catch (Exception failedValidationSaveFailureException)
+                {
+                    // We failed to run validations and failed to save the failed validation!
+                    // We were not able to process this message. We need to log the exceptions so we can debug the issue.
+                    var aggregateException = new AggregateException(
+                        "Validations failed to run and saving unsuccessful validation failed!", 
+                        new[] { validationFailedToRunException, failedValidationSaveFailureException });
+
+                    Logger.LogCritical(
+                        NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageFatalFailure,
+                        aggregateException,
+                        "Failed to process queue message");
+                }
             }
 
-            // Note that if both validations fail and saving the failure status fail, we cannot remove the message from the queue.
+            // If we failed to run validations and failed to save the failed validation, we cannot remove the message from the queue.
             if (messageWasProcessed)
             {
                 await _queue.RemoveAsync(queueMessage, token);

diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs
@@ -13,5 +13,7 @@ public static class LogEvents
 
         public static EventId StatusDeserializationFailure = new EventId(903, "Status deserialization failed!");
         public static EventId StatusDeserializationFatalFailure = new EventId(904, "Status deserialization failed, and was unable to parse id and version from filename!");
+
+        public static EventId QueueMessageFatalFailure = new EventId(905, "Failed to process queue message");
     }
 }