From 5f99bbd376c6e6fe8ecc815255f013989fd0a38e Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Thu, 15 Nov 2018 17:27:17 -0800 Subject: [PATCH 1/3] StatusAggregator should handle incident ingestion failures gracefully (#671) --- .../Export/IStatusSerializer.cs | 4 +- src/StatusAggregator/Export/StatusExporter.cs | 11 +- .../Export/StatusSerializer.cs | 4 +- src/StatusAggregator/LogEvents.cs | 1 + src/StatusAggregator/StatusAggregator.csproj | 4 +- src/StatusAggregator/Update/StatusUpdater.cs | 16 +- .../Export/StatusExporterTests.cs | 14 +- .../Export/StatusSerializerTests.cs | 9 +- .../StatusAggregator.Tests.csproj | 4 +- .../Update/StatusUpdaterTests.cs | 252 ++++++++++++++++++ 10 files changed, 298 insertions(+), 21 deletions(-) create mode 100644 tests/StatusAggregator.Tests/Update/StatusUpdaterTests.cs diff --git a/src/StatusAggregator/Export/IStatusSerializer.cs b/src/StatusAggregator/Export/IStatusSerializer.cs index fea2d31c2..5c586e36e 100644 --- a/src/StatusAggregator/Export/IStatusSerializer.cs +++ b/src/StatusAggregator/Export/IStatusSerializer.cs @@ -11,8 +11,8 @@ namespace StatusAggregator.Export public interface IStatusSerializer { /// - /// Serializes and and saves to storage with a time of . + /// Serializes and and saves to storage with a last built time of and a last updated time of . /// - Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable recentEvents); + Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable recentEvents); } } \ No newline at end of file diff --git a/src/StatusAggregator/Export/StatusExporter.cs b/src/StatusAggregator/Export/StatusExporter.cs index 467bf861d..09a83e52f 100644 --- a/src/StatusAggregator/Export/StatusExporter.cs +++ b/src/StatusAggregator/Export/StatusExporter.cs @@ -5,11 +5,14 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs.Extensions; +using StatusAggregator.Collector; +using StatusAggregator.Update; namespace StatusAggregator.Export { public class StatusExporter : IStatusExporter { + private readonly ICursor _cursor; private readonly IComponentExporter _componentExporter; private readonly IEventsExporter _eventExporter; private readonly IStatusSerializer _serializer; @@ -17,24 +20,28 @@ public class StatusExporter : IStatusExporter private readonly ILogger _logger; public StatusExporter( + ICursor cursor, IComponentExporter componentExporter, IEventsExporter eventExporter, IStatusSerializer serializer, ILogger logger) { + _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor)); _componentExporter = componentExporter ?? throw new ArgumentNullException(nameof(componentExporter)); _eventExporter = eventExporter ?? throw new ArgumentNullException(nameof(eventExporter)); _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public Task Export(DateTime cursor) + public async Task Export(DateTime cursor) { using (_logger.Scope("Exporting service status.")) { var rootComponent = _componentExporter.Export(); var recentEvents = _eventExporter.Export(cursor); - return _serializer.Serialize(cursor, rootComponent, recentEvents); + + var lastUpdated = await _cursor.Get(StatusUpdater.LastUpdatedCursorName); + await _serializer.Serialize(cursor, lastUpdated, rootComponent, recentEvents); } } } diff --git a/src/StatusAggregator/Export/StatusSerializer.cs b/src/StatusAggregator/Export/StatusSerializer.cs index d10340db5..adc85cd71 100644 --- a/src/StatusAggregator/Export/StatusSerializer.cs +++ b/src/StatusAggregator/Export/StatusSerializer.cs @@ -36,13 +36,13 @@ public StatusSerializer( _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable recentEvents) + public async Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable recentEvents) { ServiceStatus status; string statusJson; using (_logger.Scope("Serializing service status.")) { - status = new ServiceStatus(cursor, rootComponent, recentEvents); + status = new ServiceStatus(lastBuilt, lastUpdated, rootComponent, recentEvents); statusJson = JsonConvert.SerializeObject(status, Settings); } diff --git a/src/StatusAggregator/LogEvents.cs b/src/StatusAggregator/LogEvents.cs index 0709f723c..3259958f5 100644 --- a/src/StatusAggregator/LogEvents.cs +++ b/src/StatusAggregator/LogEvents.cs @@ -6,5 +6,6 @@ public static class LogEvents { public static EventId RegexFailure = new EventId(400, "Failed to parse incident using Regex."); public static EventId ManualChangeFailure = new EventId(401, "Failed to apply a manual change."); + public static EventId IncidentIngestionFailure = new EventId(402, "Failed to update incident API data."); } } diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 0fd160d6c..0cfd35052 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -160,10 +160,10 @@ 2.33.0 - 2.33.0 + 2.37.0-sb-statusfb-2205681 - 2.33.0 + 2.37.0-sb-statusfb-2205681 9.2.0 diff --git a/src/StatusAggregator/Update/StatusUpdater.cs b/src/StatusAggregator/Update/StatusUpdater.cs index affa3389d..7af2341e8 100644 --- a/src/StatusAggregator/Update/StatusUpdater.cs +++ b/src/StatusAggregator/Update/StatusUpdater.cs @@ -13,8 +13,7 @@ namespace StatusAggregator.Update { public class StatusUpdater : IStatusUpdater { - private const string ManualCursorBaseName = "manual"; - private const string IncidentCursorName = "incident"; + public const string LastUpdatedCursorName = "updated"; private readonly ICursor _cursor; private readonly IEntityCollector _incidentCollector; @@ -50,8 +49,17 @@ public async Task Update(DateTime cursor) await manualStatusChangeCollector.FetchLatest(); } - await _incidentCollector.FetchLatest(); - await _activeEventUpdater.UpdateAllAsync(cursor); + try + { + await _incidentCollector.FetchLatest(); + await _activeEventUpdater.UpdateAllAsync(cursor); + + await _cursor.Set(LastUpdatedCursorName, cursor); + } + catch (Exception e) + { + _logger.LogError(LogEvents.IncidentIngestionFailure, e, "Failed to update incident API data."); + } } } diff --git a/tests/StatusAggregator.Tests/Export/StatusExporterTests.cs b/tests/StatusAggregator.Tests/Export/StatusExporterTests.cs index d5437c077..6df7f8cc0 100644 --- a/tests/StatusAggregator.Tests/Export/StatusExporterTests.cs +++ b/tests/StatusAggregator.Tests/Export/StatusExporterTests.cs @@ -6,7 +6,9 @@ using Microsoft.Extensions.Logging; using Moq; using NuGet.Services.Status; +using StatusAggregator.Collector; using StatusAggregator.Export; +using StatusAggregator.Update; using Xunit; namespace StatusAggregator.Tests.Export @@ -29,18 +31,24 @@ public async Task ExportsAtCursor() EventExporter .Setup(x => x.Export(cursor)) .Returns(events); - + + var lastUpdated = new DateTime(2018, 9, 12); + Cursor + .Setup(x => x.Get(StatusUpdater.LastUpdatedCursorName)) + .ReturnsAsync(lastUpdated); + await Exporter.Export(cursor); Serializer .Verify( - x => x.Serialize(cursor, component, events), + x => x.Serialize(cursor, lastUpdated, component, events), Times.Once()); } } public class StatusExporterTest { + public Mock Cursor { get; } public Mock ComponentExporter { get; } public Mock EventExporter { get; } public Mock Serializer { get; } @@ -48,11 +56,13 @@ public class StatusExporterTest public StatusExporterTest() { + Cursor = new Mock(); ComponentExporter = new Mock(); EventExporter = new Mock(); Serializer = new Mock(); Exporter = new StatusExporter( + Cursor.Object, ComponentExporter.Object, EventExporter.Object, Serializer.Object, diff --git a/tests/StatusAggregator.Tests/Export/StatusSerializerTests.cs b/tests/StatusAggregator.Tests/Export/StatusSerializerTests.cs index 62d501749..bb9f91bc1 100644 --- a/tests/StatusAggregator.Tests/Export/StatusSerializerTests.cs +++ b/tests/StatusAggregator.Tests/Export/StatusSerializerTests.cs @@ -20,14 +20,15 @@ public class TheSerializeMethod : StatusSerializerTest [Fact] public async Task SerializesStatus() { - var cursor = new DateTime(2018, 9, 13); + var lastBuilt = new DateTime(2018, 11, 13); + var lastUpdated = new DateTime(2018, 9, 13); var component = new TestComponent("hi", new[] { new TestComponent("yo"), new TestComponent("what's up") }); - var events = new[] { new Event("", cursor, cursor, new[] { new Message(cursor, "howdy") }) }; + var events = new[] { new Event("", lastUpdated, lastUpdated, new[] { new Message(lastUpdated, "howdy") }) }; - var expectedStatus = new ServiceStatus(cursor, component, events); + var expectedStatus = new ServiceStatus(lastBuilt, lastUpdated, component, events); var expectedJson = JsonConvert.SerializeObject(expectedStatus, StatusSerializer.Settings); - await Serializer.Serialize(cursor, component, events); + await Serializer.Serialize(lastBuilt, lastUpdated, component, events); Container .Verify( diff --git a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj index b9ff9308a..85e5d5259 100644 --- a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj +++ b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj @@ -91,6 +91,7 @@ + @@ -111,9 +112,6 @@ 4.4.0 - - 9.2.0 - 2.4.0 diff --git a/tests/StatusAggregator.Tests/Update/StatusUpdaterTests.cs b/tests/StatusAggregator.Tests/Update/StatusUpdaterTests.cs new file mode 100644 index 000000000..87cf012f5 --- /dev/null +++ b/tests/StatusAggregator.Tests/Update/StatusUpdaterTests.cs @@ -0,0 +1,252 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using StatusAggregator.Collector; +using StatusAggregator.Update; +using Xunit; + +namespace StatusAggregator.Tests.Update +{ + public class StatusUpdaterTests + { + public class TheUpdateMethod + : StatusUpdaterTest + { + [Fact] + public async Task DoesNotUpdateCursorIfIncidentCollectorFails() + { + ManualStatusChangeCollector1 + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 12)) + .Verifiable(); + + ManualStatusChangeCollector2 + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 13)) + .Verifiable(); + + IncidentCollector + .Setup(x => x.FetchLatest()) + .ThrowsAsync(new Exception()) + .Verifiable(); + + var cursor = new DateTime(2018, 11, 12); + await Updater.Update(cursor); + + ManualStatusChangeCollector1.Verify(); + ManualStatusChangeCollector2.Verify(); + IncidentCollector.Verify(); + + ActiveEventEntityUpdater + .Verify( + x => x.UpdateAllAsync(It.IsAny()), + Times.Never()); + + Cursor + .Verify( + x => x.Set(It.IsAny(), It.IsAny()), + Times.Never()); + } + + [Fact] + public async Task DoesNotUpdateCursorIfActiveEventUpdaterFails() + { + ManualStatusChangeCollector1 + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 12)) + .Verifiable(); + + ManualStatusChangeCollector2 + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 13)) + .Verifiable(); + + IncidentCollector + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 14)) + .Verifiable(); + + var cursor = new DateTime(2018, 11, 12); + ActiveEventEntityUpdater + .Setup(x => x.UpdateAllAsync(cursor)) + .ThrowsAsync(new Exception()) + .Verifiable(); + + await Updater.Update(cursor); + + ManualStatusChangeCollector1.Verify(); + ManualStatusChangeCollector2.Verify(); + IncidentCollector.Verify(); + ActiveEventEntityUpdater.Verify(); + + Cursor + .Verify( + x => x.Set(It.IsAny(), It.IsAny()), + Times.Never()); + } + + [Fact] + public async Task UpdatesCursorIfSuccessful() + { + ManualStatusChangeCollector1 + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 12)) + .Verifiable(); + + ManualStatusChangeCollector2 + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 13)) + .Verifiable(); + + IncidentCollector + .Setup(x => x.FetchLatest()) + .ReturnsAsync(new DateTime(2018, 11, 14)) + .Verifiable(); + + var cursor = new DateTime(2018, 11, 12); + ActiveEventEntityUpdater + .Setup(x => x.UpdateAllAsync(cursor)) + .Returns(Task.CompletedTask) + .Verifiable(); + + Cursor + .Setup(x => x.Set(StatusUpdater.LastUpdatedCursorName, cursor)) + .Returns(Task.CompletedTask) + .Verifiable(); + + await Updater.Update(cursor); + + ManualStatusChangeCollector1.Verify(); + ManualStatusChangeCollector2.Verify(); + IncidentCollector.Verify(); + ActiveEventEntityUpdater.Verify(); + Cursor.Verify(); + } + } + + public class StatusUpdaterTest + { + public Mock Cursor { get; } + public Mock IncidentCollector { get; } + public Mock ManualStatusChangeCollector1 { get; } + public Mock ManualStatusChangeCollector2 { get; } + public Mock ActiveEventEntityUpdater { get; } + + public StatusUpdater Updater { get; } + + public StatusUpdaterTest() + { + Cursor = new Mock(); + IncidentCollector = CreateCollectorWithName( + IncidentEntityCollectorProcessor.IncidentsCollectorName); + ManualStatusChangeCollector1 = CreateCollectorWithName( + ManualStatusChangeCollectorProcessor.ManualCollectorNamePrefix + "1"); + ManualStatusChangeCollector2 = CreateCollectorWithName( + ManualStatusChangeCollectorProcessor.ManualCollectorNamePrefix + "2"); + ActiveEventEntityUpdater = new Mock(); + + Updater = new StatusUpdater( + Cursor.Object, + new[] + { + IncidentCollector, + ManualStatusChangeCollector1, + ManualStatusChangeCollector2 + }.Select(x => x.Object), + ActiveEventEntityUpdater.Object, + Mock.Of>()); + } + } + + public class TheConstructor + { + [Fact] + public void ThrowsWithoutCursor() + { + var incidentCollector = new Mock(); + incidentCollector + .Setup(x => x.Name) + .Returns(IncidentEntityCollectorProcessor.IncidentsCollectorName); + + Assert.Throws( + () => new StatusUpdater( + null, + new[] { incidentCollector.Object }, + Mock.Of(), + Mock.Of>())); + } + + public static IEnumerable ThrowsWithoutCollectors_Data + { + get + { + // null enumerable + yield return new object[] { typeof(ArgumentNullException), null }; + + // empty enumerable + yield return new object[] { typeof(ArgumentException), new IEntityCollector[0] }; + + // enumerable without incident collector + yield return new object[] { typeof(ArgumentException), new[] { CreateCollectorWithName("howdy").Object } }; + } + } + + [Theory] + [MemberData(nameof(ThrowsWithoutCollectors_Data))] + public void ThrowsWithoutCollectors(Type exceptionType, IEnumerable collectors) + { + Assert.Throws( + exceptionType, + () => new StatusUpdater( + Mock.Of(), + collectors, + Mock.Of(), + Mock.Of>())); + } + + [Fact] + public void ThrowsWithoutActiveEventUpdater() + { + Assert.Throws( + () => new StatusUpdater( + Mock.Of(), + new[] + { + CreateCollectorWithName(IncidentEntityCollectorProcessor.IncidentsCollectorName).Object + }, + null, + Mock.Of>())); + } + + [Fact] + public void ThrowsWithoutLogger() + { + Assert.Throws( + () => new StatusUpdater( + Mock.Of(), + new[] + { + CreateCollectorWithName(IncidentEntityCollectorProcessor.IncidentsCollectorName).Object + }, + Mock.Of(), + null)); + } + } + + private static Mock CreateCollectorWithName(string name) + { + var collector = new Mock(); + collector + .Setup(x => x.Name) + .Returns(name); + + return collector; + } + } +} From 529603829c070cc740c5b32876447cb180ba624f Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 16 Nov 2018 12:54:23 -0800 Subject: [PATCH 2/3] StatusAggregator should treat V2 Restore and Search as Active-Active components (#674) --- .../Export/ComponentExporter.cs | 2 +- .../Factory/NuGetServiceComponentFactory.cs | 8 +- .../Parse/IncidentRegexParser.cs | 2 +- src/StatusAggregator/StatusAggregator.csproj | 6 +- .../MessageChangeEventProcessorTests.cs | 4 +- .../Messages/MessageContentBuilderTests.cs | 203 ++++-------------- 6 files changed, 49 insertions(+), 176 deletions(-) diff --git a/src/StatusAggregator/Export/ComponentExporter.cs b/src/StatusAggregator/Export/ComponentExporter.cs index 66c4da59d..c89d5f000 100644 --- a/src/StatusAggregator/Export/ComponentExporter.cs +++ b/src/StatusAggregator/Export/ComponentExporter.cs @@ -71,7 +71,7 @@ public IComponent Export() foreach (var activeEntity in activeEntities) { using (_logger.Scope("Applying active entity affecting {AffectedComponentPath} of severity {AffectedComponentStatus} at {StartTime} to root component", - activeEntity.AffectedComponentPath, activeEntity.AffectedComponentStatus, activeEntity.StartTime)) + activeEntity.AffectedComponentPath, (ComponentStatus)activeEntity.AffectedComponentStatus, activeEntity.StartTime)) { var currentComponent = rootComponent.GetByPath(activeEntity.AffectedComponentPath); diff --git a/src/StatusAggregator/Factory/NuGetServiceComponentFactory.cs b/src/StatusAggregator/Factory/NuGetServiceComponentFactory.cs index 8c28b4509..8a1cabfcc 100644 --- a/src/StatusAggregator/Factory/NuGetServiceComponentFactory.cs +++ b/src/StatusAggregator/Factory/NuGetServiceComponentFactory.cs @@ -34,7 +34,7 @@ public IComponent Create() "", new IComponent[] { - new PrimarySecondaryComponent( + new ActivePassiveComponent( GalleryName, "Browsing the Gallery website", new[] @@ -55,7 +55,7 @@ public IComponent Create() new LeafComponent(GlobalRegionName, "V3 restore for users outside of China"), new LeafComponent(ChinaRegionName, "V3 restore for users inside China") }), - new PrimarySecondaryComponent( + new ActiveActiveComponent( V2ProtocolName, "Restore using the V2 API", new[] @@ -69,7 +69,7 @@ public IComponent Create() "Searching for new and existing packages in Visual Studio or the Gallery website", new[] { - new PrimarySecondaryComponent( + new ActiveActiveComponent( GlobalRegionName, "Search for packages outside China", new[] @@ -77,7 +77,7 @@ public IComponent Create() new LeafComponent(UsncInstanceName, "Primary region"), new LeafComponent(UsscInstanceName, "Backup region") }), - new PrimarySecondaryComponent( + new ActiveActiveComponent( ChinaRegionName, "Search for packages inside China", new[] diff --git a/src/StatusAggregator/Parse/IncidentRegexParser.cs b/src/StatusAggregator/Parse/IncidentRegexParser.cs index 99bd42991..64b176760 100644 --- a/src/StatusAggregator/Parse/IncidentRegexParser.cs +++ b/src/StatusAggregator/Parse/IncidentRegexParser.cs @@ -56,7 +56,7 @@ public bool TryParseIncident(Incident incident, out ParsedIncident parsedInciden return false; } - _logger.LogInformation("RegEx match result: {MatchResult}", title, match.Success); + _logger.LogInformation("RegEx match result: {MatchResult}", match.Success); return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident); } } diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 0cfd35052..ed42c751e 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -157,13 +157,13 @@ 1.1.1 - 2.33.0 + 2.38.0 - 2.37.0-sb-statusfb-2205681 + 2.38.0 - 2.37.0-sb-statusfb-2205681 + 2.38.0 9.2.0 diff --git a/tests/StatusAggregator.Tests/Messages/MessageChangeEventProcessorTests.cs b/tests/StatusAggregator.Tests/Messages/MessageChangeEventProcessorTests.cs index 68d6d985a..b430556ad 100644 --- a/tests/StatusAggregator.Tests/Messages/MessageChangeEventProcessorTests.cs +++ b/tests/StatusAggregator.Tests/Messages/MessageChangeEventProcessorTests.cs @@ -122,7 +122,7 @@ public async Task IgnoresStartMessageWhereComponentDoesntAffectStatus() public async Task CreatesStartMessageFromNullContextForHiddenComponent(ComponentStatus status) { var child = new TestComponent("child"); - var root = new PrimarySecondaryComponent("hi", "", new[] { child }); + var root = new ActivePassiveComponent("hi", "", new[] { child }); var change = new MessageChangeEvent( DefaultTimestamp, @@ -313,7 +313,7 @@ public async Task CreatesEndMessageWithContext(ComponentStatus changeStatus, Com { var child = new TestComponent("child"); child.Status = changeStatus; - var root = new PrimarySecondaryComponent("hi", "", new[] { child }); + var root = new ActiveActiveComponent("hi", "", new[] { child }); var affectedComponent = root.GetByNames(root.Name, child.Name); var change = new MessageChangeEvent( diff --git a/tests/StatusAggregator.Tests/Messages/MessageContentBuilderTests.cs b/tests/StatusAggregator.Tests/Messages/MessageContentBuilderTests.cs index a4bb6cf74..5252c27cc 100644 --- a/tests/StatusAggregator.Tests/Messages/MessageContentBuilderTests.cs +++ b/tests/StatusAggregator.Tests/Messages/MessageContentBuilderTests.cs @@ -126,253 +126,126 @@ public static IComponent CreateTestComponent(params string[] names) public class BuildsContentsSuccessfully_Data : IEnumerable { - public IEnumerable>> Data = - new Tuple>[] + /// + /// This is a map of what message template should be used given a message type and component path. + /// + public IEnumerable>> Data = + new Tuple>[] { - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName }, - ComponentStatus.Degraded, status => $"**NuGet.org is {status}.** You may encounter issues browsing the NuGet Gallery."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName }, - ComponentStatus.Degraded, status => $"**NuGet.org is no longer {status}.** You should no longer encounter any issues browsing the NuGet Gallery. Thank you for your patience."), - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName }, - ComponentStatus.Down, - status => $"**NuGet.org is {status}.** You may encounter issues browsing the NuGet Gallery."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName }, - ComponentStatus.Down, - status => $"**NuGet.org is no longer {status}.** You should no longer encounter any issues browsing the NuGet Gallery. Thank you for your patience."), - - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Degraded, - status => $"**China V3 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V3 feed from China."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Degraded, - status => $"**China V3 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V3 feed from China. Thank you for your patience."), - - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Down, status => $"**China V3 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V3 feed from China."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Down, status => $"**China V3 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V3 feed from China. Thank you for your patience."), - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Degraded, status => $"**Global V3 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V3 feed."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Degraded, status => $"**Global V3 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V3 feed. Thank you for your patience."), - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Down, - status => $"**Global V3 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V3 feed."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Down, - status => $"**Global V3 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V3 feed. Thank you for your patience."), - - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName }, - ComponentStatus.Degraded, status => $"**V3 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V3 feed."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName }, - ComponentStatus.Degraded, status => $"**V3 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V3 feed. Thank you for your patience."), - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName }, - ComponentStatus.Down, - status => $"**V3 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V3 feed."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName }, - ComponentStatus.Down, - status => $"**V3 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V3 feed. Thank you for your patience."), - - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V2ProtocolName }, - ComponentStatus.Degraded, status => $"**V2 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V2 feed."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V2ProtocolName }, - ComponentStatus.Degraded, - status => $"**V2 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V2 feed. Thank you for your patience."), - - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V2ProtocolName }, - ComponentStatus.Down, - status => $"**V2 Protocol Restore is {status}.** You may encounter issues restoring packages from NuGet.org's V2 feed."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V2ProtocolName }, - ComponentStatus.Down, status => $"**V2 Protocol Restore is no longer {status}.** You should no longer encounter any issues restoring packages from NuGet.org's V2 feed. Thank you for your patience."), - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName }, - ComponentStatus.Degraded, status => $"**Restore is {status}.** You may encounter issues restoring packages."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName }, - ComponentStatus.Degraded, status => $"**Restore is no longer {status}.** You should no longer encounter any issues restoring packages. Thank you for your patience."), - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName }, - ComponentStatus.Down, - status => $"**Restore is {status}.** You may encounter issues restoring packages."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName }, - ComponentStatus.Down, - status => $"**Restore is no longer {status}.** You should no longer encounter any issues restoring packages. Thank you for your patience."), - - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Degraded, status => $"**China Search is {status}.** You may encounter issues searching for packages from China."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Degraded, status => $"**China Search is no longer {status}.** You should no longer encounter any issues searching for packages from China. Thank you for your patience."), - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Down, - status => $"**China Search is {status}.** You may encounter issues searching for packages from China."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.ChinaRegionName }, - ComponentStatus.Down, - status => $"**China Search is no longer {status}.** You should no longer encounter any issues searching for packages from China. Thank you for your patience."), - - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Degraded, - status => $"**Global Search is {status}.** You may encounter issues searching for packages."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Degraded, - status => $"**Global Search is no longer {status}.** You should no longer encounter any issues searching for packages. Thank you for your patience."), - - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Down, status => $"**Global Search is {status}.** You may encounter issues searching for packages."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.GlobalRegionName }, - ComponentStatus.Down, status => $"**Global Search is no longer {status}.** You should no longer encounter any issues searching for packages. Thank you for your patience."), - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName }, - ComponentStatus.Degraded, status => $"**Search is {status}.** You may encounter issues searching for packages."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName }, - ComponentStatus.Degraded, status => $"**Search is no longer {status}.** You should no longer encounter any issues searching for packages. Thank you for your patience."), - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName }, - ComponentStatus.Down, - status => $"**Search is {status}.** You may encounter issues searching for packages."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName }, - ComponentStatus.Down, - status => $"**Search is no longer {status}.** You should no longer encounter any issues searching for packages. Thank you for your patience."), - - Tuple.Create>( + Tuple.Create>( MessageType.Start, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.UploadName }, - ComponentStatus.Degraded, status => $"**Package Publishing is {status}.** You may encounter issues uploading new packages."), - Tuple.Create>( + Tuple.Create>( MessageType.End, new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.UploadName }, - ComponentStatus.Degraded, - status => $"**Package Publishing is no longer {status}.** You should no longer encounter any issues uploading new packages. Thank you for your patience."), - - Tuple.Create>( - MessageType.Start, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.UploadName }, - ComponentStatus.Down, - status => $"**Package Publishing is {status}.** You may encounter issues uploading new packages."), - - Tuple.Create>( - MessageType.End, - new[] { NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.UploadName }, - ComponentStatus.Down, - status => $"**Package Publishing is no longer {status}.** You should no longer encounter any issues uploading new packages. Thank you for your patience."), - + status => $"**Package Publishing is no longer {status}.** You should no longer encounter any issues uploading new packages. Thank you for your patience.") }; + /// + /// Each scenario in needs to be tested given all possible except . + /// + private IEnumerable WrapDataWithComponentStatus(Tuple> data) + { + foreach (var status in new[] { ComponentStatus.Degraded, ComponentStatus.Down }) + { + yield return new object[] { data.Item1, data.Item2, status, data.Item3 }; + } + } + public IEnumerator GetEnumerator() => Data - .Select(t => new object[] { t.Item1, t.Item2, t.Item3, t.Item4 }) + .SelectMany(WrapDataWithComponentStatus) .GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); From 8459db8db5a91a5733da919eec9cc4208ca0ffe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Tue, 20 Nov 2018 13:29:21 -0800 Subject: [PATCH 3/3] Speed up the revalidation job by the batch size (#675) Currently, the job will increase its revalidation rate by 1 revalidation per hour each time it enqueues a batch of revalidations. As a result, the job speeds up very slowly. This change makes the job increase its revalidation rate by the batch size. Meaning, the revalidation job will increase its revalidation rate by 64 revalidations per hour if it enqueues 64 revalidations. --- .../Services/RevalidationJobStateService.cs | 3 ++- .../Services/RevalidationJobStateServiceFacts.cs | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs b/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs index 2ace054da..961e8994b 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs @@ -104,7 +104,8 @@ await _state.MaybeUpdateStateAsync(state => return false; } - var nextRate = Math.Min(_config.MaxPackageEventRate, state.DesiredPackageEventRate + 1); + var nextRate = state.DesiredPackageEventRate + _config.Queue.MaxBatchSize; + nextRate = Math.Min(_config.MaxPackageEventRate, nextRate); _logger.LogInformation( "Increasing desired package event rate to {ToRate} from {FromRate}", diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs index 7403364bf..7c7aa9f06 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs @@ -183,7 +183,7 @@ public async Task IncreasesDesiredPackageEventRateValue() await _target.IncreaseDesiredPackageEventRateAsync(); // Assert - Assert.Equal(124, state.DesiredPackageEventRate); + Assert.Equal(143, state.DesiredPackageEventRate); Assert.True(result); _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); @@ -261,6 +261,11 @@ public FactsBase() { MinPackageEventRate = 100, MaxPackageEventRate = 500, + + Queue = new RevalidationQueueConfiguration + { + MaxBatchSize = 20 + } }; _target = new RevalidationJobStateService(