Merge pull request #676 from NuGet/dev

[ReleasePrep][2018.11.20]RI of dev into master

src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs

@@ -104,7 +104,8 @@ await _state.MaybeUpdateStateAsync(state =>
                     return false;
                 }
 
-                var nextRate = Math.Min(_config.MaxPackageEventRate, state.DesiredPackageEventRate + 1);
+                var nextRate = state.DesiredPackageEventRate + _config.Queue.MaxBatchSize;
+                nextRate = Math.Min(_config.MaxPackageEventRate, nextRate);
 
                 _logger.LogInformation(
                     "Increasing desired package event rate to {ToRate} from {FromRate}",

src/StatusAggregator/Export/ComponentExporter.cs

@@ -71,7 +71,7 @@ public IComponent Export()
                 foreach (var activeEntity in activeEntities)
                 {
                     using (_logger.Scope("Applying active entity affecting {AffectedComponentPath} of severity {AffectedComponentStatus} at {StartTime} to root component",
-                        activeEntity.AffectedComponentPath, activeEntity.AffectedComponentStatus, activeEntity.StartTime))
+                        activeEntity.AffectedComponentPath, (ComponentStatus)activeEntity.AffectedComponentStatus, activeEntity.StartTime))
                     {
                         var currentComponent = rootComponent.GetByPath(activeEntity.AffectedComponentPath);
 

src/StatusAggregator/Export/IStatusSerializer.cs

@@ -11,8 +11,8 @@ namespace StatusAggregator.Export
     public interface IStatusSerializer
     {
         /// <summary>
-        /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a time of <paramref name="cursor"/>.
+        /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a last built time of <paramref name="lastBuilt"/> and a last updated time of <paramref name="lastUpdated"/>.
         /// </summary>
-        Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents);
+        Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents);
     }
 }

src/StatusAggregator/Export/StatusExporter.cs

@@ -5,36 +5,43 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Jobs.Extensions;
+using StatusAggregator.Collector;
+using StatusAggregator.Update;
 
 namespace StatusAggregator.Export
 {
     public class StatusExporter : IStatusExporter
     {
+        private readonly ICursor _cursor;
         private readonly IComponentExporter _componentExporter;
         private readonly IEventsExporter _eventExporter;
         private readonly IStatusSerializer _serializer;
 
         private readonly ILogger<StatusExporter> _logger;
 
         public StatusExporter(
+            ICursor cursor,
             IComponentExporter componentExporter,
             IEventsExporter eventExporter,
             IStatusSerializer serializer,
             ILogger<StatusExporter> logger)
         {
+            _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));
             _componentExporter = componentExporter ?? throw new ArgumentNullException(nameof(componentExporter));
             _eventExporter = eventExporter ?? throw new ArgumentNullException(nameof(eventExporter));
             _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public Task Export(DateTime cursor)
+        public async Task Export(DateTime cursor)
         {
             using (_logger.Scope("Exporting service status."))
             {
                 var rootComponent = _componentExporter.Export();
                 var recentEvents = _eventExporter.Export(cursor);
-                return _serializer.Serialize(cursor, rootComponent, recentEvents);
+
+                var lastUpdated = await _cursor.Get(StatusUpdater.LastUpdatedCursorName);
+                await _serializer.Serialize(cursor, lastUpdated, rootComponent, recentEvents);
             }
         }
     }

src/StatusAggregator/Export/StatusSerializer.cs

@@ -36,13 +36,13 @@ public StatusSerializer(
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents)
+        public async Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents)
         {
             ServiceStatus status;
             string statusJson;
             using (_logger.Scope("Serializing service status."))
             {
-                status = new ServiceStatus(cursor, rootComponent, recentEvents);
+                status = new ServiceStatus(lastBuilt, lastUpdated, rootComponent, recentEvents);
                 statusJson = JsonConvert.SerializeObject(status, Settings);
             }
 

src/StatusAggregator/Factory/NuGetServiceComponentFactory.cs

@@ -34,7 +34,7 @@ public IComponent Create()
                 "",
                 new IComponent[]
                 {
-                    new PrimarySecondaryComponent(
+                    new ActivePassiveComponent(
                         GalleryName,
                         "Browsing the Gallery website",
                         new[]
@@ -55,7 +55,7 @@ public IComponent Create()
                                     new LeafComponent(GlobalRegionName, "V3 restore for users outside of China"),
                                     new LeafComponent(ChinaRegionName, "V3 restore for users inside China")
                                 }),
-                            new PrimarySecondaryComponent(
+                            new ActiveActiveComponent(
                                 V2ProtocolName,
                                 "Restore using the V2 API",
                                 new[]
@@ -69,15 +69,15 @@ public IComponent Create()
                         "Searching for new and existing packages in Visual Studio or the Gallery website",
                         new[]
                         {
-                            new PrimarySecondaryComponent(
+                            new ActiveActiveComponent(
                                 GlobalRegionName,
                                 "Search for packages outside China",
                                 new[]
                                 {
                                     new LeafComponent(UsncInstanceName, "Primary region"),
                                     new LeafComponent(UsscInstanceName, "Backup region")
                                 }),
-                            new PrimarySecondaryComponent(
+                            new ActiveActiveComponent(
                                 ChinaRegionName,
                                 "Search for packages inside China",
                                 new[]

src/StatusAggregator/LogEvents.cs

@@ -6,5 +6,6 @@ public static class LogEvents
     {
         public static EventId RegexFailure = new EventId(400, "Failed to parse incident using Regex.");
         public static EventId ManualChangeFailure = new EventId(401, "Failed to apply a manual change.");
+        public static EventId IncidentIngestionFailure = new EventId(402, "Failed to update incident API data.");
     }
 }

src/StatusAggregator/Parse/IncidentRegexParser.cs

@@ -56,7 +56,7 @@ public bool TryParseIncident(Incident incident, out ParsedIncident parsedInciden
                     return false;
                 }
 
-                _logger.LogInformation("RegEx match result: {MatchResult}", title, match.Success);
+                _logger.LogInformation("RegEx match result: {MatchResult}", match.Success);
                 return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident);
             }
         }

src/StatusAggregator/StatusAggregator.csproj

@@ -157,13 +157,13 @@
       <Version>1.1.1</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Incidents">
-      <Version>2.33.0</Version>
+      <Version>2.38.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.33.0</Version>
+      <Version>2.38.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status.Table">
-      <Version>2.33.0</Version>
+      <Version>2.38.0</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
       <Version>9.2.0</Version>

src/StatusAggregator/Update/StatusUpdater.cs

@@ -13,8 +13,7 @@ namespace StatusAggregator.Update
 {
     public class StatusUpdater : IStatusUpdater
     {
-        private const string ManualCursorBaseName = "manual";
-        private const string IncidentCursorName = "incident";
+        public const string LastUpdatedCursorName = "updated";
 
         private readonly ICursor _cursor;
         private readonly IEntityCollector _incidentCollector;
@@ -50,8 +49,17 @@ public async Task Update(DateTime cursor)
                     await manualStatusChangeCollector.FetchLatest();
                 }
 
-                await _incidentCollector.FetchLatest();
-                await _activeEventUpdater.UpdateAllAsync(cursor);
+                try
+                {
+                    await _incidentCollector.FetchLatest();
+                    await _activeEventUpdater.UpdateAllAsync(cursor);
+
+                    await _cursor.Set(LastUpdatedCursorName, cursor);
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(LogEvents.IncidentIngestionFailure, e, "Failed to update incident API data.");
+                }
             }
         }
 

tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs

@@ -183,7 +183,7 @@ public async Task IncreasesDesiredPackageEventRateValue()
                 await _target.IncreaseDesiredPackageEventRateAsync();
 
                 // Assert
-                Assert.Equal(124, state.DesiredPackageEventRate);
+                Assert.Equal(143, state.DesiredPackageEventRate);
                 Assert.True(result);
 
                 _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny<Func<RevalidationState, bool>>()), Times.Once);
@@ -261,6 +261,11 @@ public FactsBase()
                 {
                     MinPackageEventRate = 100,
                     MaxPackageEventRate = 500,
+
+                    Queue = new RevalidationQueueConfiguration
+                    {
+                        MaxBatchSize = 20
+                    }
                 };
 
                 _target = new RevalidationJobStateService(

tests/StatusAggregator.Tests/Export/StatusExporterTests.cs

@@ -6,7 +6,9 @@
 using Microsoft.Extensions.Logging;
 using Moq;
 using NuGet.Services.Status;
+using StatusAggregator.Collector;
 using StatusAggregator.Export;
+using StatusAggregator.Update;
 using Xunit;
 
 namespace StatusAggregator.Tests.Export
@@ -29,30 +31,38 @@ public async Task ExportsAtCursor()
                 EventExporter
                     .Setup(x => x.Export(cursor))
                     .Returns(events);
-
+
+                var lastUpdated = new DateTime(2018, 9, 12);
+                Cursor
+                    .Setup(x => x.Get(StatusUpdater.LastUpdatedCursorName))
+                    .ReturnsAsync(lastUpdated);
+
                 await Exporter.Export(cursor);
 
                 Serializer
                     .Verify(
-                        x => x.Serialize(cursor, component, events),
+                        x => x.Serialize(cursor, lastUpdated, component, events),
                         Times.Once());
             }
         }
 
         public class StatusExporterTest
         {
+            public Mock<ICursor> Cursor { get; }
             public Mock<IComponentExporter> ComponentExporter { get; }
             public Mock<IEventsExporter> EventExporter { get; }
             public Mock<IStatusSerializer> Serializer { get; }
             public StatusExporter Exporter { get; }
 
             public StatusExporterTest()
             {
+                Cursor = new Mock<ICursor>();
                 ComponentExporter = new Mock<IComponentExporter>();
                 EventExporter = new Mock<IEventsExporter>();
                 Serializer = new Mock<IStatusSerializer>();
 
                 Exporter = new StatusExporter(
+                    Cursor.Object,
                     ComponentExporter.Object,
                     EventExporter.Object,
                     Serializer.Object,

tests/StatusAggregator.Tests/Export/StatusSerializerTests.cs

@@ -20,14 +20,15 @@ public class TheSerializeMethod : StatusSerializerTest
             [Fact]
             public async Task SerializesStatus()
             {
-                var cursor = new DateTime(2018, 9, 13);
+                var lastBuilt = new DateTime(2018, 11, 13);
+                var lastUpdated = new DateTime(2018, 9, 13);
                 var component = new TestComponent("hi", new[] { new TestComponent("yo"), new TestComponent("what's up") });
-                var events = new[] { new Event("", cursor, cursor, new[] { new Message(cursor, "howdy") }) };
+                var events = new[] { new Event("", lastUpdated, lastUpdated, new[] { new Message(lastUpdated, "howdy") }) };
 
-                var expectedStatus = new ServiceStatus(cursor, component, events);
+                var expectedStatus = new ServiceStatus(lastBuilt, lastUpdated, component, events);
                 var expectedJson = JsonConvert.SerializeObject(expectedStatus, StatusSerializer.Settings);
 
-                await Serializer.Serialize(cursor, component, events);
+                await Serializer.Serialize(lastBuilt, lastUpdated, component, events);
 
                 Container
                     .Verify(

tests/StatusAggregator.Tests/Messages/MessageChangeEventProcessorTests.cs

@@ -122,7 +122,7 @@ public async Task IgnoresStartMessageWhereComponentDoesntAffectStatus()
             public async Task CreatesStartMessageFromNullContextForHiddenComponent(ComponentStatus status)
             {
                 var child = new TestComponent("child");
-                var root = new PrimarySecondaryComponent("hi", "", new[] { child });
+                var root = new ActivePassiveComponent("hi", "", new[] { child });
 
                 var change = new MessageChangeEvent(
                     DefaultTimestamp,
@@ -313,7 +313,7 @@ public async Task CreatesEndMessageWithContext(ComponentStatus changeStatus, Com
             {
                 var child = new TestComponent("child");
                 child.Status = changeStatus;
-                var root = new PrimarySecondaryComponent("hi", "", new[] { child });
+                var root = new ActiveActiveComponent("hi", "", new[] { child });
 
                 var affectedComponent = root.GetByNames<IComponent>(root.Name, child.Name);
                 var change = new MessageChangeEvent(