Status - add export logic for two-layer aggregation (#566)

src/StatusAggregator/Container/ContainerWrapper.cs

@@ -0,0 +1,29 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace StatusAggregator.Container
+{
+    public class ContainerWrapper : IContainerWrapper
+    {
+        private readonly CloudBlobContainer _container;
+
+        public ContainerWrapper(CloudBlobContainer container)
+        {
+            _container = container;
+        }
+
+        public Task CreateIfNotExistsAsync()
+        {
+            return _container.CreateIfNotExistsAsync();
+        }
+
+        public Task SaveBlobAsync(string name, string contents)
+        {
+            var blob = _container.GetBlockBlobReference(name);
+            return blob.UploadTextAsync(contents);
+        }
+    }
+}

src/StatusAggregator/Container/IContainerWrapper.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace StatusAggregator.Container
+{
+    /// <summary>
+    /// Simple wrapper for <see cref="CloudBlobContainer"/> that exists for unit-testing.
+    /// </summary>
+    public interface IContainerWrapper
+    {
+        Task CreateIfNotExistsAsync();
+
+        Task SaveBlobAsync(string name, string contents);
+    }
+}

src/StatusAggregator/Export/ComponentExporter.cs

@@ -0,0 +1,91 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
+using NuGet.Services.Status;
+using NuGet.Services.Status.Table;
+using StatusAggregator.Factory;
+using StatusAggregator.Table;
+
+namespace StatusAggregator.Export
+{
+    public class ComponentExporter : IComponentExporter
+    {
+        private readonly ITableWrapper _table;
+        private readonly IComponentFactory _factory;
+
+        private readonly ILogger<ComponentExporter> _logger;
+
+        public ComponentExporter(
+            ITableWrapper table,
+            IComponentFactory factory,
+            ILogger<ComponentExporter> logger)
+        {
+            _table = table ?? throw new ArgumentNullException(nameof(table));
+            _factory = factory ?? throw new ArgumentNullException(nameof(factory));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public IComponent Export()
+        {
+            using (_logger.Scope("Exporting active entities to component."))
+            {
+                var rootComponent = _factory.Create();
+
+                // Apply the active entities to the component tree.
+                var activeEvents = _table
+                    .GetActiveEntities<EventEntity>()
+                    .ToList()
+                    .Where(e => 
+                        _table
+                            .GetChildEntities<MessageEntity, EventEntity>(e)
+                            .ToList()
+                            .Any())
+                    .ToList();
+
+                _logger.LogInformation("Found {EventCount} active events with messages.", activeEvents.Count);
+
+                var activeIncidentGroups = activeEvents
+                    .SelectMany(e => 
+                        _table
+                            .GetChildEntities<IncidentGroupEntity, EventEntity>(e)
+                            .Where(i => i.IsActive)
+                            .ToList())
+                    .ToList();
+
+                _logger.LogInformation("Found {GroupCount} active incident groups linked to active events with messages.", activeIncidentGroups.Count);
+
+                var activeEntities = activeIncidentGroups
+                    .Concat<IComponentAffectingEntity>(activeEvents)
+                    // Only apply entities with a non-Up status.
+                    .Where(e => e.AffectedComponentStatus != (int)ComponentStatus.Up)
+                    // If multiple events are affecting a single region, the event with the highest severity should affect the component.
+                    .GroupBy(e => e.AffectedComponentPath)
+                    .Select(g => g.OrderByDescending(e => e.AffectedComponentStatus).First())
+                    .ToList();
+
+                _logger.LogInformation("Active entities affect {PathCount} distinct subcomponents.", activeEntities.Count);
+                foreach (var activeEntity in activeEntities)
+                {
+                    using (_logger.Scope("Applying active entity affecting {AffectedComponentPath} of severity {AffectedComponentStatus} at {StartTime} to root component",
+                        activeEntity.AffectedComponentPath, activeEntity.AffectedComponentStatus, activeEntity.StartTime))
+                    {
+                        var currentComponent = rootComponent.GetByPath(activeEntity.AffectedComponentPath);
+
+                        if (currentComponent == null)
+                        {
+                            throw new InvalidOperationException($"Couldn't find component with path {activeEntity.AffectedComponentPath} corresponding to active entities.");
+                        }
+
+                        currentComponent.Status = (ComponentStatus)activeEntity.AffectedComponentStatus;
+                    }
+                }
+
+                return rootComponent;
+            }
+        }
+    }
+}

src/StatusAggregator/Export/EventExporter.cs

@@ -0,0 +1,55 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
+using NuGet.Services.Status;
+using NuGet.Services.Status.Table;
+using StatusAggregator.Table;
+
+namespace StatusAggregator.Export
+{
+    public class EventExporter : IEventExporter
+    {
+        private readonly ITableWrapper _table;
+        private readonly ILogger<EventExporter> _logger;
+
+        public EventExporter(
+            ITableWrapper table,
+            ILogger<EventExporter> logger)
+        {
+            _table = table ?? throw new ArgumentNullException(nameof(table));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public Event Export(EventEntity eventEntity)
+        {
+            using (_logger.Scope("Exporting event {EventRowKey}.", eventEntity.RowKey))
+            {
+                var messages = _table.GetChildEntities<MessageEntity, EventEntity>(eventEntity)
+                    .ToList()
+                    // Don't show empty messages.
+                    .Where(m => !string.IsNullOrEmpty(m.Contents))
+                    .ToList();
+
+                _logger.LogInformation("Event has {MessageCount} messages that are not empty.", messages.Count);
+
+                if (!messages.Any())
+                {
+                    return null;
+                }
+
+                return new Event(
+                    eventEntity.AffectedComponentPath, 
+                    eventEntity.StartTime,
+                    eventEntity.EndTime, 
+                    messages
+                        .OrderBy(m => m.Time)
+                        .Select(m => new Message(m.Time, m.Contents)));
+            }
+        }
+    }
+}

src/StatusAggregator/Export/EventsExporter.cs

@@ -0,0 +1,46 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Status;
+using NuGet.Services.Status.Table;
+using StatusAggregator.Table;
+
+namespace StatusAggregator.Export
+{
+    public class EventsExporter : IEventsExporter
+    {
+        private readonly TimeSpan _eventVisibilityPeriod;
+
+        private readonly ITableWrapper _table;
+        private readonly IEventExporter _exporter;
+
+        private readonly ILogger<EventsExporter> _logger;
+
+        public EventsExporter(
+            ITableWrapper table,
+            IEventExporter exporter,
+            StatusAggregatorConfiguration configuration,
+            ILogger<EventsExporter> logger)
+        {
+            _table = table ?? throw new ArgumentNullException(nameof(table));
+            _exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
+            _eventVisibilityPeriod = TimeSpan.FromDays(configuration?.EventVisibilityPeriodDays ?? throw new ArgumentNullException(nameof(configuration)));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public IEnumerable<Event> Export(DateTime cursor)
+        {
+            return _table
+                .CreateQuery<EventEntity>()
+                .Where(e => e.IsActive || (e.EndTime >= cursor - _eventVisibilityPeriod))
+                .ToList()
+                .Select(_exporter.Export)
+                .Where(e => e != null)
+                .ToList();
+        }
+    }
+}

src/StatusAggregator/Export/IComponentExporter.cs

@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using NuGet.Services.Status;
+
+namespace StatusAggregator.Export
+{
+    public interface IComponentExporter
+    {
+        /// <summary>
+        /// Exports the status of the current active entities to an <see cref="IComponent"/>.
+        /// </summary>
+        IComponent Export();
+    }
+}

src/StatusAggregator/Export/IEventExporter.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using NuGet.Services.Status;
+using NuGet.Services.Status.Table;
+
+namespace StatusAggregator.Export
+{
+    public interface IEventExporter
+    {
+        /// <summary>
+        /// Exports <paramref name="eventEntity"/> as a <see cref="Event"/>. If it should not be exported, returns <c>null</c>.
+        /// </summary>
+        Event Export(EventEntity eventEntity);
+    }
+}

src/StatusAggregator/Export/IEventsExporter.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using NuGet.Services.Status;
+
+namespace StatusAggregator.Export
+{
+    public interface IEventsExporter
+    {
+        /// <summary>
+        /// Exports recent events.
+        /// </summary>
+        IEnumerable<Event> Export(DateTime cursor);
+    }
+}

src/StatusAggregator/IStatusExporter.cs → src/StatusAggregator/Export/IStatusExporter.cs

@@ -1,16 +1,17 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using NuGet.Services.Status;
+using System;
 using System.Threading.Tasks;
 
-namespace StatusAggregator
+namespace StatusAggregator.Export
 {
     public interface IStatusExporter
     {
         /// <summary>
         /// Builds a <see cref="ServiceStatus"/> and exports it to public storage so that it can be consumed by other services.
         /// </summary>
-        Task<ServiceStatus> Export();
+        Task Export(DateTime cursor);
     }
-}
+}

src/StatusAggregator/Export/IStatusSerializer.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using NuGet.Services.Status;
+
+namespace StatusAggregator.Export
+{
+    public interface IStatusSerializer
+    {
+        /// <summary>
+        /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a time of <paramref name="cursor"/>.
+        /// </summary>
+        Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents);
+    }
+}

src/StatusAggregator/Export/StatusExporter.cs

@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
+
+namespace StatusAggregator.Export
+{
+    public class StatusExporter : IStatusExporter
+    {
+        private readonly IComponentExporter _componentExporter;
+        private readonly IEventsExporter _eventExporter;
+        private readonly IStatusSerializer _serializer;
+
+        private readonly ILogger<StatusExporter> _logger;
+
+        public StatusExporter(
+            IComponentExporter componentExporter,
+            IEventsExporter eventExporter,
+            IStatusSerializer serializer,
+            ILogger<StatusExporter> logger)
+        {
+            _componentExporter = componentExporter ?? throw new ArgumentNullException(nameof(componentExporter));
+            _eventExporter = eventExporter ?? throw new ArgumentNullException(nameof(eventExporter));
+            _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public Task Export(DateTime cursor)
+        {
+            using (_logger.Scope("Exporting service status."))
+            {
+                var rootComponent = _componentExporter.Export();
+                var recentEvents = _eventExporter.Export(cursor);
+                return _serializer.Serialize(cursor, rootComponent, recentEvents);
+            }
+        }
+    }
+}